From adabf8e089fa0690ce76dafc68e465075d8f010a Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 27 Mar 2026 15:02:40 +0800 Subject: [PATCH] feat(datasource): implement data source management functionality - Added a new API for managing data sources, including CRUD operations and connection validation. - Introduced UI components for data source settings, sync logs, and editor dialog, enhancing user experience in managing external data sources. - Updated localization files to support new data source features in multiple languages. - Implemented cron expression humanization for better scheduling visibility. - Enhanced knowledge base editor to include data source management options. --- frontend/src/api/datasource/index.ts | 112 +++ .../src/components/AgentShareSettings.vue | 2 +- frontend/src/i18n/locales/en-US.ts | 163 ++++ frontend/src/i18n/locales/ko-KR.ts | 121 +++ frontend/src/i18n/locales/ru-RU.ts | 123 ++- frontend/src/i18n/locales/zh-CN.ts | 163 ++++ frontend/src/utils/cronHumanize.ts | 46 + .../knowledge/KnowledgeBaseEditorModal.vue | 18 +- .../settings/DataSourceEditorDialog.vue | 834 ++++++++++++++++++ .../knowledge/settings/DataSourceSettings.vue | 489 ++++++++++ .../knowledge/settings/DataSourceSyncLogs.vue | 446 ++++++++++ .../knowledge/settings/GraphSettings.vue | 8 +- .../knowledge/settings/KBAdvancedSettings.vue | 8 +- .../knowledge/settings/KBChunkingSettings.vue | 8 +- .../knowledge/settings/KBModelConfig.vue | 8 +- .../knowledge/settings/KBParserSettings.vue | 8 +- .../knowledge/settings/KBShareSettings.vue | 6 +- .../knowledge/settings/KBStorageSettings.vue | 8 +- .../application/repository/datasource_repo.go | 235 +++++ internal/application/repository/message.go | 2 +- internal/application/repository/tag.go | 6 +- internal/application/repository/tenant.go | 9 +- .../application/service/datasource_service.go | 713 +++++++++++++++ internal/container/container.go | 49 +- .../CONNECTOR_IMPLEMENTATION_GUIDE.md | 497 +++++++++++ internal/datasource/README.md | 260 ++++++ internal/datasource/connector.go | 221 +++++ .../datasource/connector/feishu/client.go | 451 ++++++++++ .../datasource/connector/feishu/connector.go | 376 ++++++++ .../connector/feishu/connector_test.go | 817 +++++++++++++++++ internal/datasource/connector/feishu/types.go | 206 +++++ internal/datasource/errors.go | 32 + internal/datasource/scheduler.go | 208 +++++ internal/datasource/scheduler_test.go | 356 ++++++++ internal/handler/datasource.go | 549 ++++++++++++ internal/router/router.go | 35 + internal/router/sync_task.go | 2 + internal/router/task.go | 4 + internal/types/datasource.go | 413 +++++++++ internal/types/extract_graph.go | 1 + internal/types/interfaces/datasource.go | 100 +++ internal/types/interfaces/knowledge.go | 4 + .../000028_datasource_tables.down.sql | 6 + .../versioned/000028_datasource_tables.up.sql | 78 ++ 44 files changed, 8159 insertions(+), 42 deletions(-) create mode 100644 frontend/src/api/datasource/index.ts create mode 100644 frontend/src/utils/cronHumanize.ts create mode 100644 frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue create mode 100644 frontend/src/views/knowledge/settings/DataSourceSettings.vue create mode 100644 frontend/src/views/knowledge/settings/DataSourceSyncLogs.vue create mode 100644 internal/application/repository/datasource_repo.go create mode 100644 internal/application/service/datasource_service.go create mode 100644 internal/datasource/CONNECTOR_IMPLEMENTATION_GUIDE.md create mode 100644 internal/datasource/README.md create mode 100644 internal/datasource/connector.go create mode 100644 internal/datasource/connector/feishu/client.go create mode 100644 internal/datasource/connector/feishu/connector.go create mode 100644 internal/datasource/connector/feishu/connector_test.go create mode 100644 internal/datasource/connector/feishu/types.go create mode 100644 internal/datasource/errors.go create mode 100644 internal/datasource/scheduler.go create mode 100644 internal/datasource/scheduler_test.go create mode 100644 internal/handler/datasource.go create mode 100644 internal/types/datasource.go create mode 100644 internal/types/interfaces/datasource.go create mode 100644 migrations/versioned/000028_datasource_tables.down.sql create mode 100644 migrations/versioned/000028_datasource_tables.up.sql diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts new file mode 100644 index 00000000..8681dc25 --- /dev/null +++ b/frontend/src/api/datasource/index.ts @@ -0,0 +1,112 @@ +import { get, post, put, del } from '../../utils/request' + +// --- Types --- + +export interface DataSource { + id: string + tenant_id: number + knowledge_base_id: string + name: string + type: string + config: any + sync_schedule: string + sync_mode: 'incremental' | 'full' + status: 'active' | 'paused' | 'error' + conflict_strategy: 'overwrite' | 'skip' + sync_deletions: boolean + last_sync_at: string | null + last_sync_result: any + error_message: string + created_at: string + updated_at: string + latest_sync_log?: SyncLog +} + +export interface SyncLog { + id: string + data_source_id: string + status: 'running' | 'success' | 'partial' | 'failed' | 'canceled' + started_at: string + finished_at: string | null + items_total: number + items_created: number + items_updated: number + items_deleted: number + items_skipped: number + items_failed: number + error_message: string +} + +export interface ConnectorMeta { + type: string + name: string + description: string + icon: string + priority: number + auth_type: string + capabilities: string[] +} + +export interface Resource { + external_id: string + name: string + type: string + description: string + url: string + parent_id?: string +} + +// --- API calls --- + +export function getConnectorTypes() { + return get('/api/v1/datasource/types') +} + +export function listDataSources(kbId: string) { + return get(`/api/v1/datasource?kb_id=${encodeURIComponent(kbId)}`) +} + +export function getDataSource(id: string) { + return get(`/api/v1/datasource/${id}`) +} + +export function createDataSource(data: Partial) { + return post('/api/v1/datasource', data) +} + +export function updateDataSource(id: string, data: Partial) { + return put(`/api/v1/datasource/${id}`, data) +} + +export function deleteDataSource(id: string) { + return del(`/api/v1/datasource/${id}`) +} + +export function validateConnection(id: string) { + return post(`/api/v1/datasource/${id}/validate`, {}) +} + +// Validate credentials without persisting (for "Test Connection" during creation) +export function validateCredentials(type: string, credentials: Record) { + return post('/api/v1/datasource/validate-credentials', { type, credentials }) +} + +export function listResources(id: string) { + return get(`/api/v1/datasource/${id}/resources`) +} + +export function triggerSync(id: string) { + return post(`/api/v1/datasource/${id}/sync`, {}) +} + +export function pauseDataSource(id: string) { + return post(`/api/v1/datasource/${id}/pause`, {}) +} + +export function resumeDataSource(id: string) { + return post(`/api/v1/datasource/${id}/resume`, {}) +} + +export function getSyncLogs(id: string, limit = 20, offset = 0) { + return get(`/api/v1/datasource/${id}/logs?limit=${limit}&offset=${offset}`) +} diff --git a/frontend/src/components/AgentShareSettings.vue b/frontend/src/components/AgentShareSettings.vue index 187c7b50..6bfe131c 100644 --- a/frontend/src/components/AgentShareSettings.vue +++ b/frontend/src/components/AgentShareSettings.vue @@ -272,7 +272,7 @@ defineExpose({ loadShares }) margin-bottom: 16px; .shares-title { font-family: "PingFang SC"; - font-size: 14px; + font-size: 15px; font-weight: 500; color: var(--td-text-color-primary); } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 4038e553..8734bcd4 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1432,6 +1432,7 @@ export default { faq: 'FAQ Settings', graph: 'Knowledge Graph', multimodal: 'Multimodal', + datasource: 'Data Sources', share: 'Sharing' }, basic: { @@ -3344,4 +3345,166 @@ export default { yesterday: 'Yesterday', daysAgo: '{days} days ago', }, + datasource: { + title: 'Data Sources', + description: 'Configure external data sources to sync content into this knowledge base', + add: 'Add Data Source', + addFirst: 'Add your first data source', + empty: 'No data sources configured', + edit: 'Edit', + delete: 'Delete', + deleteConfirm: 'Delete this data source? Synced knowledge will not be removed.', + deleteSuccess: 'Data source deleted', + deleteFailed: 'Failed to delete', + syncNow: 'Sync Now', + syncTriggered: 'Sync task submitted', + syncFailed: 'Failed to trigger sync', + pause: 'Pause', + resume: 'Resume', + paused: 'Paused', + resumed: 'Resumed', + pauseFailed: 'Failed to pause', + logs: 'Logs', + lastSync: 'Last sync', + lastStatus: 'Result', + schedule: 'Schedule', + syncModeLabel: 'Sync mode', + syncMode: { + incremental: 'Incremental', + full: 'Full', + }, + status: { + active: 'Connected', + paused: 'Paused', + error: 'Error', + }, + createTitle: 'Add Data Source', + editTitle: 'Edit Data Source', + nameLabel: 'Name', + namePlaceholder: 'Enter data source name', + testConnection: 'Test Connection', + testSuccess: 'Connection successful', + testFailed: 'Connection failed', + connected: 'Connected', + connectionFailed: 'Connection failed', + isRequired: 'is required', + resourceHint: 'Select the spaces or folders to sync', + resourceLoadFailed: 'Failed to load resources', + noResources: 'No wiki spaces found', + noResourcesDesc: 'The app needs wiki access via a group chat to fetch content', + retryLoadResources: 'Retry', + guideStep1: 'Create a group chat in Feishu, then add your app as a bot in the group settings', + guideStep2: 'Open wiki "Settings" > "Member Settings" > "Add Member", search for the group chat and add it', + guideStep3: 'Ensure the group chat role is at least "Can Read", then come back and click Retry', + permissionDocLink: 'View Feishu wiki permission docs', + syncScheduleLabel: 'Sync schedule', + conflictLabel: 'Conflict strategy', + conflict: { + overwrite: 'Overwrite', + skip: 'Skip existing', + }, + syncDeletions: 'Sync deletions (remove knowledge when deleted at source)', + createAndSync: 'Create & Sync Now', + createSuccess: 'Data source created', + updateSuccess: 'Data source updated', + saveFailed: 'Failed to save', + step: { + selectType: 'Select Type', + credentials: 'Credentials', + resources: 'Resources', + strategy: 'Strategy', + }, + syncHistory: 'Sync History', + noLogs: 'No sync records yet', + logStatus: { + running: 'Syncing', + success: 'Success', + partial: 'Partial', + failed: 'Failed', + canceled: 'Canceled', + }, + logMetric: { + total: 'Total', + created: 'created', + updated: 'updated', + deleted: 'deleted', + skipped: 'skipped', + failed: 'failed', + }, + logSummary: { + total: 'Runs', + success: 'Success', + failed: 'Failed', + items: 'Items', + }, + logLatest: 'Latest', + logDetail: { + startTime: 'Started', + endTime: 'Finished', + duration: 'Duration', + }, + connector: { + feishu: 'Feishu', + notion: 'Notion', + yuque: 'Yuque', + wecom_doc: 'WeCom Docs', + }, + connectorDesc: { + feishu: 'Sync documents, spreadsheets and files from Feishu Wiki', + notion: 'Sync pages and databases from Notion', + yuque: 'Sync documents from Yuque knowledge bases', + wecom_doc: 'Sync documents from WeCom Docs', + }, + field: { + appId: 'App ID', + appSecret: 'App Secret', + integrationToken: 'Integration Token', + apiToken: 'API Token', + corpId: 'Corp ID', + corpSecret: 'Corp Secret', + }, + comingSoon: 'Coming soon', + docHint: 'Get credentials at:', + pleaseTestFirst: 'Please test connection first', + prereqTitle: 'Complete these steps before connecting (all required)', + prereqBarText: 'First time? Click to see Feishu app setup guide', + prereqBotBrief: 'Add "Bot" capability to your app', + prereqBotDesc: 'Open Platform > Add App Capability > Bot > create version and publish', + prereqBotLabel: '1. Add "Bot" capability to your app (otherwise the app won\'t appear in wiki member search):', + prereqBotStep1: 'Open your app in Feishu Open Platform, click "Add App Capability" in the left menu', + prereqBotStep2: 'Select "Bot" and confirm', + prereqBotStep3: 'Create a new version and publish (or test in your test organization)', + prereqPermBrief: 'Grant API permissions', + prereqPermLabel: '2. Grant these API permissions:', + prereqOpenConsole: 'Open Feishu Developer Console', + prereqMemberBrief: 'Add app to wiki via group chat', + prereqMemberDesc: 'Create group chat > add app as bot > add group chat as wiki member', + prereqMemberLabel: '3. Add the app to a wiki space via group chat (wiki cannot search for apps directly):', + back: 'Back', + next: 'Next', + save: 'Save', + schedule30min: 'Every 30 min', + schedule1h: 'Every hour', + schedule6h: 'Every 6 hours', + schedule12h: 'Every 12 hours', + schedule24h: 'Daily', + scheduleHuman: { + '30min': 'Every 30 min', + '1h': 'Hourly', + '6h': 'Every 6 hours', + '12h': 'Every 12 hours', + '24h': 'Daily', + }, + addCard: 'Add Data Source', + wikiSpace: 'Wiki Space', + neverSynced: 'Never synced', + justNow: 'Just now', + minutesAgo: '{n}m ago', + hoursAgo: '{n}h ago', + daysAgo: '{n}d ago', + syncResultCreated: '+{n}', + syncResultUpdated: '~{n}', + syncResultDeleted: '-{n}', + moreActions: 'More', + }, } diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 23d6cbf9..cf32266a 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1915,6 +1915,7 @@ export default { graph: "지식 그래프", share: "공유관리", storage: "스토리지 엔진", + datasource: "데이터 소스", }, basic: { title: "기본 정보", @@ -3390,4 +3391,124 @@ export default { yesterday: "어제", daysAgo: "{days}일 전", }, + datasource: { + title: "데이터 소스 관리", + description: "외부 데이터 소스를 구성하여 콘텐츠를 지식베이스에 자동 동기화", + add: "데이터 소스 추가", + addFirst: "첫 번째 데이터 소스 추가", + empty: "구성된 데이터 소스가 없습니다", + edit: "편집", + delete: "삭제", + deleteConfirm: "이 데이터 소스를 삭제하시겠습니까? 동기화된 지식은 삭제되지 않습니다.", + deleteSuccess: "데이터 소스가 삭제되었습니다", + deleteFailed: "삭제 실패", + syncNow: "지금 동기화", + syncTriggered: "동기화 작업이 제출되었습니다", + syncFailed: "동기화 트리거 실패", + pause: "일시정지", + resume: "재개", + paused: "일시정지됨", + resumed: "재개됨", + pauseFailed: "일시정지 실패", + logs: "로그", + lastSync: "마지막 동기화", + lastStatus: "결과", + schedule: "동기화 주기", + syncModeLabel: "동기화 모드", + syncMode: { + incremental: "증분 동기화", + full: "전체 동기화", + }, + status: { + active: "연결됨", + paused: "일시정지", + error: "오류", + }, + createTitle: "데이터 소스 추가", + editTitle: "데이터 소스 편집", + nameLabel: "이름", + namePlaceholder: "데이터 소스 이름 입력", + testConnection: "연결 테스트", + testSuccess: "연결 성공", + testFailed: "연결 실패", + connected: "연결됨", + connectionFailed: "연결 실패", + isRequired: "은(는) 필수입니다", + resourceHint: "동기화할 공간/폴더를 선택하세요", + resourceLoadFailed: "리소스 목록 로드 실패", + noResources: "동기화 가능한 위키 공간을 찾을 수 없습니다", + noResourcesHint: "먼저 데이터 소스를 저장하고 권한 설정을 확인하세요", + permissionHint: "앱이 위키 공간의 멤버로 추가되어야 콘텐츠에 접근할 수 있습니다:", + permissionStep1: "페이슈를 열고 동기화할 위키 공간으로 이동", + permissionStep2: "오른쪽 상단 '...' > '설정' > '멤버 관리' 클릭", + permissionStep3: "페이슈 앱을 검색하여 위키 공간 멤버로 추가", + permissionDocLink: "페이슈 위키 권한 설정 문서 보기", + syncScheduleLabel: "동기화 주기", + conflictLabel: "충돌 전략", + conflict: { + overwrite: "덮어쓰기", + skip: "기존 항목 건너뛰기", + }, + syncDeletions: "삭제 동기화 (소스에서 삭제 시 지식베이스에서도 삭제)", + createAndSync: "생성 후 즉시 동기화", + createSuccess: "데이터 소스가 생성되었습니다", + updateSuccess: "데이터 소스가 업데이트되었습니다", + saveFailed: "저장 실패", + step: { + selectType: "유형 선택", + credentials: "인증 정보", + resources: "리소스 선택", + strategy: "동기화 전략", + }, + syncHistory: "동기화 이력", + noLogs: "동기화 기록이 없습니다", + logStatus: { + running: "동기화 중", + success: "성공", + partial: "부분 성공", + failed: "실패", + canceled: "취소됨", + }, + logMetric: { + total: "총계", + failed: "실패", + }, + connector: { + feishu: "페이슈 (Feishu)", + notion: "Notion", + confluence: "Confluence", + yuque: "위큐 (Yuque)", + github: "GitHub", + web_crawler: "웹 크롤러", + }, + connectorDesc: { + feishu: "페이슈 위키에서 문서, 스프레드시트, 파일 동기화", + notion: "Notion에서 페이지 및 데이터베이스 동기화", + confluence: "Confluence에서 스페이스 및 페이지 동기화", + yuque: "위큐 지식베이스에서 문서 동기화", + github: "GitHub 리포지토리에서 문서 및 위키 동기화", + web_crawler: "사이트맵을 통해 웹사이트 크롤링", + }, + field: { + appId: "App ID", + appSecret: "App Secret", + integrationToken: "Integration Token", + baseUrl: "Base URL", + email: "이메일", + apiToken: "API Token", + personalToken: "Personal Access Token", + sitemapUrl: "사이트맵 URL", + }, + comingSoon: "곧 지원 예정", + docHint: "다음에서 인증 정보를 받으세요:", + pleaseTestFirst: "먼저 연결을 테스트하세요", + back: "이전", + next: "다음", + save: "저장", + schedule30min: "30분마다", + schedule1h: "매시간", + schedule6h: "6시간마다", + schedule12h: "12시간마다", + schedule24h: "매일", + }, }; diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 9f2f00a3..39097579 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1600,6 +1600,7 @@ export default { faq: 'FAQ настройки', graph: 'Граф знаний', storage: 'Storage Engine', + datasource: 'Источники данных', share: 'Sharing' }, basic: { @@ -3296,5 +3297,125 @@ export default { sharedKBs: ' knowledge bases', sharedAgents: ' agents' } - } + }, + datasource: { + title: 'Источники данных', + description: 'Настройте внешние источники данных для автоматической синхронизации контента', + add: 'Добавить источник', + addFirst: 'Добавить первый источник данных', + empty: 'Источники данных не настроены', + edit: 'Редактировать', + delete: 'Удалить', + deleteConfirm: 'Удалить этот источник данных? Синхронизированные знания не будут удалены.', + deleteSuccess: 'Источник данных удалён', + deleteFailed: 'Не удалось удалить', + syncNow: 'Синхронизировать', + syncTriggered: 'Задача синхронизации отправлена', + syncFailed: 'Не удалось запустить синхронизацию', + pause: 'Пауза', + resume: 'Возобновить', + paused: 'Приостановлено', + resumed: 'Возобновлено', + pauseFailed: 'Не удалось приостановить', + logs: 'Журнал', + lastSync: 'Последняя синхронизация', + lastStatus: 'Результат', + schedule: 'Расписание', + syncModeLabel: 'Режим синхронизации', + syncMode: { + incremental: 'Инкрементная', + full: 'Полная', + }, + status: { + active: 'Подключено', + paused: 'Приостановлено', + error: 'Ошибка', + }, + createTitle: 'Добавить источник данных', + editTitle: 'Редактировать источник данных', + nameLabel: 'Название', + namePlaceholder: 'Введите название источника данных', + testConnection: 'Проверить подключение', + testSuccess: 'Подключение успешно', + testFailed: 'Подключение не удалось', + connected: 'Подключено', + connectionFailed: 'Подключение не удалось', + isRequired: 'обязательно для заполнения', + resourceHint: 'Выберите пространства или папки для синхронизации', + resourceLoadFailed: 'Не удалось загрузить список ресурсов', + noResources: 'Пространства вики не найдены', + noResourcesHint: 'Сначала сохраните источник данных и проверьте права доступа', + permissionHint: 'Приложение должно быть добавлено как участник пространства вики для доступа к контенту:', + permissionStep1: 'Откройте Feishu и перейдите в пространство вики для синхронизации', + permissionStep2: 'Нажмите "..." > "Настройки" > "Управление участниками"', + permissionStep3: 'Найдите и добавьте приложение Feishu как участника пространства', + permissionDocLink: 'Документация по настройке прав доступа', + syncScheduleLabel: 'Расписание синхронизации', + conflictLabel: 'Стратегия конфликтов', + conflict: { + overwrite: 'Перезаписать', + skip: 'Пропустить существующие', + }, + syncDeletions: 'Синхронизировать удаления (удалять знания при удалении в источнике)', + createAndSync: 'Создать и синхронизировать', + createSuccess: 'Источник данных создан', + updateSuccess: 'Источник данных обновлён', + saveFailed: 'Не удалось сохранить', + step: { + selectType: 'Выберите тип', + credentials: 'Учётные данные', + resources: 'Ресурсы', + strategy: 'Стратегия', + }, + syncHistory: 'История синхронизации', + noLogs: 'Нет записей синхронизации', + logStatus: { + running: 'Синхронизация', + success: 'Успешно', + partial: 'Частично', + failed: 'Ошибка', + canceled: 'Отменено', + }, + logMetric: { + total: 'Всего', + failed: 'ошибок', + }, + connector: { + feishu: 'Feishu (Фэйшу)', + notion: 'Notion', + confluence: 'Confluence', + yuque: 'Yuque (Юйцюэ)', + github: 'GitHub', + web_crawler: 'Веб-краулер', + }, + connectorDesc: { + feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki', + notion: 'Синхронизация страниц и баз данных из Notion', + confluence: 'Синхронизация пространств и страниц из Confluence', + yuque: 'Синхронизация документов из баз знаний Yuque', + github: 'Синхронизация документов и вики из репозиториев GitHub', + web_crawler: 'Сканирование веб-сайтов через Sitemap', + }, + field: { + appId: 'App ID', + appSecret: 'App Secret', + integrationToken: 'Integration Token', + baseUrl: 'Base URL', + email: 'Email', + apiToken: 'API Token', + personalToken: 'Personal Access Token', + sitemapUrl: 'Sitemap URL', + }, + comingSoon: 'Скоро', + docHint: 'Получить учётные данные можно здесь:', + pleaseTestFirst: 'Сначала проверьте подключение', + back: 'Назад', + next: 'Далее', + save: 'Сохранить', + schedule30min: 'Каждые 30 мин', + schedule1h: 'Каждый час', + schedule6h: 'Каждые 6 часов', + schedule12h: 'Каждые 12 часов', + schedule24h: 'Ежедневно', + }, } diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index bababa13..acebfaba 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1904,6 +1904,7 @@ export default { faq: "FAQ 设置", graph: "知识图谱", multimodal: "多模态", + datasource: "数据源", share: "共享管理", }, basic: { @@ -3342,4 +3343,166 @@ export default { yesterday: "昨天", daysAgo: "{days} 天前", }, + datasource: { + title: "数据源管理", + description: "配置外部数据源,自动同步内容到知识库", + add: "添加数据源", + addFirst: "添加第一个数据源", + empty: "暂无数据源", + edit: "编辑", + delete: "删除", + deleteConfirm: "确定删除该数据源?同步的知识不会被删除。", + deleteSuccess: "数据源已删除", + deleteFailed: "删除失败", + syncNow: "立即同步", + syncTriggered: "同步任务已提交", + syncFailed: "同步触发失败", + pause: "暂停", + resume: "恢复", + paused: "已暂停", + resumed: "已恢复", + pauseFailed: "暂停失败", + logs: "日志", + lastSync: "上次同步", + lastStatus: "同步结果", + schedule: "同步频率", + syncModeLabel: "同步模式", + syncMode: { + incremental: "增量同步", + full: "全量同步", + }, + status: { + active: "已连接", + paused: "已暂停", + error: "异常", + }, + createTitle: "添加数据源", + editTitle: "编辑数据源", + nameLabel: "名称", + namePlaceholder: "输入数据源名称", + testConnection: "测试连接", + testSuccess: "连接成功", + testFailed: "连接失败", + connected: "已连接", + connectionFailed: "连接失败", + isRequired: "为必填项", + resourceHint: "选择要同步的内容空间/文件夹", + resourceLoadFailed: "加载资源列表失败", + noResources: "未找到可同步的知识库空间", + noResourcesDesc: "应用需要通过群聊获得知识库访问权限才能拉取内容", + retryLoadResources: "重新加载", + guideStep1: "在飞书中创建一个群聊,在群设置「群机器人」中添加你的应用", + guideStep2: "打开知识库「设置」→「成员设置」→ 添加成员,搜索该群聊名称并添加", + guideStep3: "确保群聊角色至少为「可阅读」,然后回到这里点重新加载", + permissionDocLink: "查看飞书知识库权限配置文档", + syncScheduleLabel: "同步频率", + conflictLabel: "冲突策略", + conflict: { + overwrite: "覆盖更新", + skip: "跳过已存在", + }, + syncDeletions: "同步删除(源端删除时同步删除知识库中的条目)", + createAndSync: "创建并立即同步", + createSuccess: "数据源创建成功", + updateSuccess: "数据源更新成功", + saveFailed: "保存失败", + step: { + selectType: "选择类型", + credentials: "配置凭证", + resources: "选择范围", + strategy: "同步策略", + }, + syncHistory: "同步历史", + noLogs: "暂无同步记录", + logStatus: { + running: "同步中", + success: "成功", + partial: "部分成功", + failed: "失败", + canceled: "已取消", + }, + logMetric: { + total: "总计", + created: "新增", + updated: "更新", + deleted: "删除", + skipped: "跳过", + failed: "失败", + }, + logSummary: { + total: "总次数", + success: "成功", + failed: "失败", + items: "同步条目", + }, + logLatest: "最新", + logDetail: { + startTime: "开始时间", + endTime: "结束时间", + duration: "耗时", + }, + connector: { + feishu: "飞书", + notion: "Notion", + yuque: "语雀", + wecom_doc: "企微文档", + }, + connectorDesc: { + feishu: "同步飞书知识库中的文档、表格、文件", + notion: "同步 Notion 中的页面和数据库", + yuque: "同步语雀知识库中的文档", + wecom_doc: "同步企业微信文档中的内容", + }, + field: { + appId: "App ID", + appSecret: "App Secret", + integrationToken: "Integration Token", + apiToken: "API Token", + corpId: "Corp ID", + corpSecret: "Corp Secret", + }, + comingSoon: "即将支持", + docHint: "在以下地址获取凭证:", + pleaseTestFirst: "请先测试连接", + prereqTitle: "使用前请先完成以下配置(缺一不可)", + prereqBarText: "首次使用?点击查看飞书应用配置指引", + prereqBotBrief: "为应用添加「机器人」能力", + prereqBotDesc: "开放平台 → 添加应用能力 → 机器人 → 创建版本并发布", + prereqBotLabel: "1. 为应用添加「机器人」能力(否则知识库中搜不到该应用):", + prereqBotStep1: "在飞书开放平台打开你的应用,点击左侧「添加应用能力」", + prereqBotStep2: "选择「机器人」并确认添加", + prereqBotStep3: "创建新版本并发布(或在测试企业中调试)", + prereqPermBrief: "开通 API 权限", + prereqPermLabel: "2. 为应用开通以下 API 权限:", + prereqOpenConsole: "前往飞书开放平台配置", + prereqMemberBrief: "通过群聊添加为知识库成员", + prereqMemberDesc: "创建群聊 → 添加应用为群机器人 → 将群聊添加为知识库成员", + prereqMemberLabel: "3. 通过群聊将应用添加为知识库成员(知识库无法直接搜到应用):", + back: "上一步", + next: "下一步", + save: "保存", + schedule30min: "每 30 分钟", + schedule1h: "每小时", + schedule6h: "每 6 小时", + schedule12h: "每 12 小时", + schedule24h: "每天", + scheduleHuman: { + '30min': "每 30 分钟", + '1h': "每小时", + '6h': "每 6 小时", + '12h': "每 12 小时", + '24h': "每天", + }, + addCard: "添加数据源", + wikiSpace: "知识库空间", + neverSynced: "未同步", + justNow: "刚刚", + minutesAgo: "{n} 分钟前", + hoursAgo: "{n} 小时前", + daysAgo: "{n} 天前", + syncResultCreated: "+{n}", + syncResultUpdated: "~{n}", + syncResultDeleted: "-{n}", + moreActions: "更多", + }, }; diff --git a/frontend/src/utils/cronHumanize.ts b/frontend/src/utils/cronHumanize.ts new file mode 100644 index 00000000..7dc5334b --- /dev/null +++ b/frontend/src/utils/cronHumanize.ts @@ -0,0 +1,46 @@ +// cronHumanize maps known cron schedule presets to human-readable i18n keys. +// Falls back to the raw cron expression for unknown patterns. + +const CRON_PRESET_MAP: Record = { + '0 */30 * * * *': 'datasource.scheduleHuman.30min', + '0 0 * * * *': 'datasource.scheduleHuman.1h', + '0 0 */6 * * *': 'datasource.scheduleHuman.6h', + '0 0 */12 * * *': 'datasource.scheduleHuman.12h', + '0 0 2 * * *': 'datasource.scheduleHuman.24h', +} + +/** + * Convert a cron expression to a human-readable string. + * Uses i18n translate function for known presets, raw expression otherwise. + */ +export function humanizeCron(cron: string, t: (key: string) => string): string { + const key = CRON_PRESET_MAP[cron] + if (key) return t(key) + return cron || '--' +} + +/** + * Format a timestamp as relative time (e.g. "3小时前", "2天前"). + * Falls back to locale date string for timestamps older than 30 days. + */ +export function relativeTime(ts: string | null, t: (key: string, params?: Record) => string): string { + if (!ts) return t('datasource.neverSynced') + const then = new Date(ts).getTime() + if (isNaN(then)) return t('datasource.neverSynced') + const now = Date.now() + const diffMs = now - then + + // Future time or just happened (within 60s) + if (diffMs < 60000) return t('datasource.justNow') + + const minutes = Math.floor(diffMs / 60000) + if (minutes < 60) return t('datasource.minutesAgo', { n: minutes }) + + const hours = Math.floor(diffMs / 3600000) + if (hours < 24) return t('datasource.hoursAgo', { n: hours }) + + const days = Math.floor(diffMs / 86400000) + if (days < 30) return t('datasource.daysAgo', { n: days }) + + return new Date(ts).toLocaleDateString() +} diff --git a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue index 9ae1ff57..317dc480 100644 --- a/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue +++ b/frontend/src/views/knowledge/KnowledgeBaseEditorModal.vue @@ -212,6 +212,11 @@ /> + +
+ +
+
@@ -250,6 +255,7 @@ import KBAdvancedSettings from './settings/KBAdvancedSettings.vue' import ModelSelector from '@/components/ModelSelector.vue' import GraphSettings from './settings/GraphSettings.vue' import KBShareSettings from './settings/KBShareSettings.vue' +import DataSourceSettings from './settings/DataSourceSettings.vue' import { useI18n } from 'vue-i18n' const uiStore = useUIStore() @@ -292,8 +298,12 @@ const navItems = computed(() => { { key: 'multimodal', icon: 'image', label: t('knowledgeEditor.sidebar.multimodal') }, { key: 'advanced', icon: 'setting', label: t('knowledgeEditor.sidebar.advanced') } ) + // 数据源管理:仅文档类型知识库 + 编辑模式 + if (props.mode === 'edit' && props.kbId) { + items.push({ key: 'datasource', icon: 'cloud-download', label: t('knowledgeEditor.sidebar.datasource') }) + } } - // 只在编辑模式下显示共享标签页 + // 共享设置:编辑模式下所有类型都可用 if (props.mode === 'edit' && props.kbId) { items.push({ key: 'share', icon: 'share', label: t('knowledgeEditor.sidebar.share') }) } @@ -810,7 +820,7 @@ watch( .settings-modal { position: relative; width: 90vw; - max-width: 1100px; + max-width: 1000px; height: 85vh; max-height: 750px; background: var(--td-bg-color-container); @@ -944,7 +954,7 @@ watch( .section-title { margin: 0 0 8px 0; font-family: "PingFang SC"; - font-size: 16px; + font-size: 20px; font-weight: 600; color: var(--td-text-color-primary); } @@ -974,7 +984,7 @@ watch( display: block; margin-bottom: 8px; font-family: "PingFang SC"; - font-size: 14px; + font-size: 15px; font-weight: 500; color: var(--td-text-color-primary); diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue new file mode 100644 index 00000000..7cc79eda --- /dev/null +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -0,0 +1,834 @@ + + + + + diff --git a/frontend/src/views/knowledge/settings/DataSourceSettings.vue b/frontend/src/views/knowledge/settings/DataSourceSettings.vue new file mode 100644 index 00000000..c5e16860 --- /dev/null +++ b/frontend/src/views/knowledge/settings/DataSourceSettings.vue @@ -0,0 +1,489 @@ + + + + + diff --git a/frontend/src/views/knowledge/settings/DataSourceSyncLogs.vue b/frontend/src/views/knowledge/settings/DataSourceSyncLogs.vue new file mode 100644 index 00000000..036fea21 --- /dev/null +++ b/frontend/src/views/knowledge/settings/DataSourceSyncLogs.vue @@ -0,0 +1,446 @@ + + + + + diff --git a/frontend/src/views/knowledge/settings/GraphSettings.vue b/frontend/src/views/knowledge/settings/GraphSettings.vue index e04d43c3..c6e56c7f 100644 --- a/frontend/src/views/knowledge/settings/GraphSettings.vue +++ b/frontend/src/views/knowledge/settings/GraphSettings.vue @@ -611,8 +611,8 @@ onMounted(async () => { } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; label { @@ -632,8 +632,8 @@ onMounted(async () => { } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; justify-content: flex-end; align-items: center; diff --git a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue index 9512c64b..86450f60 100644 --- a/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue +++ b/frontend/src/views/knowledge/settings/KBAdvancedSettings.vue @@ -129,8 +129,8 @@ const handleQuestionGenerationChange = () => { } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; label { @@ -157,8 +157,8 @@ const handleQuestionGenerationChange = () => { } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; justify-content: flex-end; align-items: center; diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index 528ec4fa..93773e8d 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -250,8 +250,8 @@ const emitUpdate = () => { } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; label { @@ -271,8 +271,8 @@ const emitUpdate = () => { } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; justify-content: flex-end; align-items: center; diff --git a/frontend/src/views/knowledge/settings/KBModelConfig.vue b/frontend/src/views/knowledge/settings/KBModelConfig.vue index eccc6d18..0b835bba 100644 --- a/frontend/src/views/knowledge/settings/KBModelConfig.vue +++ b/frontend/src/views/knowledge/settings/KBModelConfig.vue @@ -153,8 +153,8 @@ const handleAddModel = (subSection: string) => { } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; label { @@ -179,8 +179,8 @@ const handleAddModel = (subSection: string) => { } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; justify-content: flex-end; align-items: flex-start; diff --git a/frontend/src/views/knowledge/settings/KBParserSettings.vue b/frontend/src/views/knowledge/settings/KBParserSettings.vue index c737d13f..983190cb 100644 --- a/frontend/src/views/knowledge/settings/KBParserSettings.vue +++ b/frontend/src/views/knowledge/settings/KBParserSettings.vue @@ -335,8 +335,8 @@ watch(() => props.parserEngineRules, (v) => { } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; .group-label { @@ -386,8 +386,8 @@ watch(() => props.parserEngineRules, (v) => { } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; flex-direction: column; align-items: flex-end; diff --git a/frontend/src/views/knowledge/settings/KBShareSettings.vue b/frontend/src/views/knowledge/settings/KBShareSettings.vue index 2e370f07..0b8c9f14 100644 --- a/frontend/src/views/knowledge/settings/KBShareSettings.vue +++ b/frontend/src/views/knowledge/settings/KBShareSettings.vue @@ -315,7 +315,7 @@ onMounted(async () => { .section-title { margin: 0 0 8px 0; font-family: "PingFang SC"; - font-size: 16px; + font-size: 20px; font-weight: 600; color: var(--td-text-color-primary); } @@ -340,7 +340,7 @@ onMounted(async () => { display: block; margin-bottom: 8px; font-family: "PingFang SC"; - font-size: 14px; + font-size: 15px; font-weight: 500; color: var(--td-text-color-primary); } @@ -382,7 +382,7 @@ onMounted(async () => { .shares-title { font-family: "PingFang SC"; - font-size: 14px; + font-size: 15px; font-weight: 500; color: var(--td-text-color-primary); } diff --git a/frontend/src/views/knowledge/settings/KBStorageSettings.vue b/frontend/src/views/knowledge/settings/KBStorageSettings.vue index b6f3659f..c3ec4c8b 100644 --- a/frontend/src/views/knowledge/settings/KBStorageSettings.vue +++ b/frontend/src/views/knowledge/settings/KBStorageSettings.vue @@ -208,8 +208,8 @@ onMounted(load) } .setting-info { - flex: 1; - max-width: 65%; + flex: 0 0 40%; + max-width: 40%; padding-right: 24px; label { @@ -229,8 +229,8 @@ onMounted(load) } .setting-control { - flex-shrink: 0; - min-width: 280px; + flex: 0 0 55%; + max-width: 55%; display: flex; flex-direction: column; align-items: flex-start; diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go new file mode 100644 index 00000000..a888b18a --- /dev/null +++ b/internal/application/repository/datasource_repo.go @@ -0,0 +1,235 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// DataSourceRepository provides data access for data sources +type DataSourceRepository struct { + db *gorm.DB +} + +// NewDataSourceRepository creates a new data source repository +func NewDataSourceRepository(db *gorm.DB) interfaces.DataSourceRepository { + return &DataSourceRepository{db: db} +} + +// Create inserts a new data source record +func (r *DataSourceRepository) Create(ctx context.Context, ds *types.DataSource) error { + if ds == nil { + return errors.New("data source is nil") + } + if err := r.db.WithContext(ctx).Create(ds).Error; err != nil { + return err + } + return nil +} + +// FindByID retrieves a data source by ID +func (r *DataSourceRepository) FindByID(ctx context.Context, id string) (*types.DataSource, error) { + if id == "" { + return nil, errors.New("id is empty") + } + var ds types.DataSource + if err := r.db.WithContext(ctx). + Where("id = ?", id). + Where("deleted_at IS NULL"). + First(&ds).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New("data source not found") + } + return nil, err + } + return &ds, nil +} + +// FindByKnowledgeBase lists all data sources for a knowledge base +func (r *DataSourceRepository) FindByKnowledgeBase(ctx context.Context, kbID string) ([]*types.DataSource, error) { + if kbID == "" { + return nil, errors.New("knowledge base id is empty") + } + var dataSources []*types.DataSource + if err := r.db.WithContext(ctx). + Where("knowledge_base_id = ?", kbID). + Where("deleted_at IS NULL"). + Order("created_at DESC"). + Find(&dataSources).Error; err != nil { + return nil, err + } + return dataSources, nil +} + +// Update updates an existing data source +func (r *DataSourceRepository) Update(ctx context.Context, ds *types.DataSource) error { + if ds == nil { + return errors.New("data source is nil") + } + if ds.ID == "" { + return errors.New("data source id is empty") + } + if err := r.db.WithContext(ctx). + Model(ds). + Updates(ds).Error; err != nil { + return err + } + return nil +} + +// Delete performs a soft delete +func (r *DataSourceRepository) Delete(ctx context.Context, id string) error { + if id == "" { + return errors.New("id is empty") + } + if err := r.db.WithContext(ctx). + Model(&types.DataSource{}). + Where("id = ?", id). + Update("deleted_at", gorm.Expr("NOW()")).Error; err != nil { + return err + } + return nil +} + +// FindActive retrieves all active data sources (used for scheduling) +func (r *DataSourceRepository) FindActive(ctx context.Context) ([]*types.DataSource, error) { + var dataSources []*types.DataSource + if err := r.db.WithContext(ctx). + Where("status = ?", types.DataSourceStatusActive). + Where("deleted_at IS NULL"). + Where("sync_schedule != ''"). + Order("created_at DESC"). + Find(&dataSources).Error; err != nil { + return nil, err + } + return dataSources, nil +} + +// SyncLogRepository provides data access for sync logs +type SyncLogRepository struct { + db *gorm.DB +} + +// NewSyncLogRepository creates a new sync log repository +func NewSyncLogRepository(db *gorm.DB) interfaces.SyncLogRepository { + return &SyncLogRepository{db: db} +} + +// Create inserts a new sync log entry +func (r *SyncLogRepository) Create(ctx context.Context, log *types.SyncLog) error { + if log == nil { + return errors.New("sync log is nil") + } + if err := r.db.WithContext(ctx).Create(log).Error; err != nil { + return err + } + return nil +} + +// FindByID retrieves a sync log by ID +func (r *SyncLogRepository) FindByID(ctx context.Context, id string) (*types.SyncLog, error) { + if id == "" { + return nil, errors.New("id is empty") + } + var log types.SyncLog + if err := r.db.WithContext(ctx). + Where("id = ?", id). + First(&log).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New("sync log not found") + } + return nil, err + } + return &log, nil +} + +// FindByDataSource lists sync logs for a data source with pagination +func (r *SyncLogRepository) FindByDataSource(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) { + if dsID == "" { + return nil, errors.New("data source id is empty") + } + if limit <= 0 { + limit = 10 + } + if offset < 0 { + offset = 0 + } + var logs []*types.SyncLog + if err := r.db.WithContext(ctx). + Where("data_source_id = ?", dsID). + Order("started_at DESC"). + Limit(limit). + Offset(offset). + Find(&logs).Error; err != nil { + return nil, err + } + return logs, nil +} + +// FindLatest retrieves the most recent sync log for a data source +func (r *SyncLogRepository) FindLatest(ctx context.Context, dsID string) (*types.SyncLog, error) { + if dsID == "" { + return nil, errors.New("data source id is empty") + } + var log types.SyncLog + if err := r.db.WithContext(ctx). + Where("data_source_id = ?", dsID). + Order("started_at DESC"). + Limit(1). + First(&log).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, err + } + return &log, nil +} + +// HasRunningSync checks if a data source has any sync currently in "running" status. +func (r *SyncLogRepository) HasRunningSync(ctx context.Context, dsID string) (bool, error) { + if dsID == "" { + return false, errors.New("data source id is empty") + } + var count int64 + if err := r.db.WithContext(ctx). + Model(&types.SyncLog{}). + Where("data_source_id = ?", dsID). + Where("status = ?", types.SyncLogStatusRunning). + Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +// Update updates an existing sync log entry +func (r *SyncLogRepository) Update(ctx context.Context, log *types.SyncLog) error { + if log == nil { + return errors.New("sync log is nil") + } + if log.ID == "" { + return errors.New("sync log id is empty") + } + if err := r.db.WithContext(ctx). + Model(log). + Updates(log).Error; err != nil { + return err + } + return nil +} + +// CleanupOldLogs deletes sync logs older than the retention period +func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays int) error { + if retentionDays <= 0 { + retentionDays = 30 + } + // Delete logs older than the retention period + if err := r.db.WithContext(ctx). + Where("started_at < NOW() - INTERVAL ? DAY", retentionDays). + Delete(&types.SyncLog{}).Error; err != nil { + return err + } + return nil +} diff --git a/internal/application/repository/message.go b/internal/application/repository/message.go index 9d85bc6b..22d83ac3 100644 --- a/internal/application/repository/message.go +++ b/internal/application/repository/message.go @@ -168,7 +168,7 @@ func (r *messageRepository) SearchMessagesByKeyword( Joins("INNER JOIN sessions ON sessions.id = messages.session_id AND sessions.deleted_at IS NULL"). Where("sessions.tenant_id = ?", tenantID). Where("messages.deleted_at IS NULL"). - Where("messages.content ILIKE ?", "%"+keyword+"%") + Where("messages.content ILIKE ?", "%"+escapeLikeKeyword(keyword)+"%") if len(sessionIDs) > 0 { query = query.Where("messages.session_id IN ?", sessionIDs) diff --git a/internal/application/repository/tag.go b/internal/application/repository/tag.go index f1c98702..3290b4b2 100644 --- a/internal/application/repository/tag.go +++ b/internal/application/repository/tag.go @@ -107,7 +107,8 @@ func (r *knowledgeTagRepository) ListByKB( baseQuery := r.db.WithContext(ctx).Model(&types.KnowledgeTag{}). Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID) if keyword != "" { - baseQuery = baseQuery.Where("name LIKE ?", "%"+keyword+"%") + escaped := escapeLikeKeyword(keyword) + baseQuery = baseQuery.Where("name LIKE ?", "%"+escaped+"%") } if err := baseQuery.Count(&total).Error; err != nil { @@ -117,7 +118,8 @@ func (r *knowledgeTagRepository) ListByKB( dataQuery := r.db.WithContext(ctx). Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID) if keyword != "" { - dataQuery = dataQuery.Where("name LIKE ?", "%"+keyword+"%") + escaped := escapeLikeKeyword(keyword) + dataQuery = dataQuery.Where("name LIKE ?", "%"+escaped+"%") } var tags []*types.KnowledgeTag diff --git a/internal/application/repository/tenant.go b/internal/application/repository/tenant.go index d374ce73..b5978279 100644 --- a/internal/application/repository/tenant.go +++ b/internal/application/repository/tenant.go @@ -63,14 +63,13 @@ func (r *tenantRepository) SearchTenants(ctx context.Context, keyword string, te // Build search conditions if tenantID > 0 && keyword != "" { - // When both tenantID and keyword are provided, use OR to match either - query = query.Where("id = ? OR name LIKE ? OR description LIKE ?", tenantID, "%"+keyword+"%", "%"+keyword+"%") + escaped := escapeLikeKeyword(keyword) + query = query.Where("id = ? OR name LIKE ? OR description LIKE ?", tenantID, "%"+escaped+"%", "%"+escaped+"%") } else if tenantID > 0 { - // Filter by tenant ID only query = query.Where("id = ?", tenantID) } else if keyword != "" { - // Filter by keyword only (search in name and description) - query = query.Where("name LIKE ? OR description LIKE ?", "%"+keyword+"%", "%"+keyword+"%") + escaped := escapeLikeKeyword(keyword) + query = query.Where("name LIKE ? OR description LIKE ?", "%"+escaped+"%", "%"+escaped+"%") } // Count total diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go new file mode 100644 index 00000000..705cb850 --- /dev/null +++ b/internal/application/service/datasource_service.go @@ -0,0 +1,713 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "mime/multipart" + "net/textproto" + "time" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" +) + +// DataSourceService implements the DataSourceService interface +type DataSourceService struct { + dsRepo interfaces.DataSourceRepository + syncLogRepo interfaces.SyncLogRepository + knowledgeService interfaces.KnowledgeService + kbService interfaces.KnowledgeBaseService + taskEnqueuer interfaces.TaskEnqueuer + connectorRegistry *datasource.ConnectorRegistry + scheduler *datasource.Scheduler + tenantRepo interfaces.TenantRepository + tagService interfaces.KnowledgeTagService +} + +// NewDataSourceService creates a new data source service +func NewDataSourceService( + dsRepo interfaces.DataSourceRepository, + syncLogRepo interfaces.SyncLogRepository, + knowledgeService interfaces.KnowledgeService, + kbService interfaces.KnowledgeBaseService, + taskEnqueuer interfaces.TaskEnqueuer, + connectorRegistry *datasource.ConnectorRegistry, + scheduler *datasource.Scheduler, + tenantRepo interfaces.TenantRepository, + tagService interfaces.KnowledgeTagService, +) interfaces.DataSourceService { + return &DataSourceService{ + dsRepo: dsRepo, + syncLogRepo: syncLogRepo, + knowledgeService: knowledgeService, + kbService: kbService, + taskEnqueuer: taskEnqueuer, + connectorRegistry: connectorRegistry, + scheduler: scheduler, + tenantRepo: tenantRepo, + tagService: tagService, + } +} + +// CreateDataSource creates a new data source configuration +func (s *DataSourceService) CreateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) { + if ds == nil { + return nil, datasource.ErrDataSourceInvalid + } + + // Validate knowledge base exists + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, ds.KnowledgeBaseID) + if err != nil || kb == nil { + return nil, datasource.ErrKnowledgeBaseNotFound + } + + // Validate connector type + _, err = s.connectorRegistry.Get(ds.Type) + if err != nil { + return nil, err + } + + // Validate configuration + if err := s.validateDataSourceConfig(ctx, ds); err != nil { + return nil, err + } + + // Create in database + if err := s.dsRepo.Create(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to create data source: %v", err) + return nil, err + } + + // Register cron schedule if configured + if ds.SyncSchedule != "" && ds.Status == types.DataSourceStatusActive { + if err := s.scheduler.AddOrUpdate(ds); err != nil { + logger.Warnf(ctx, "failed to register cron for ds=%s: %v", ds.ID, err) + } + } + + logger.Infof(ctx, "data source created: id=%s type=%s kb=%s", ds.ID, ds.Type, ds.KnowledgeBaseID) + return ds, nil +} + +// GetDataSource retrieves a data source by ID +func (s *DataSourceService) GetDataSource(ctx context.Context, id string) (*types.DataSource, error) { + ds, err := s.dsRepo.FindByID(ctx, id) + if err != nil { + return nil, err + } + return ds, nil +} + +// ListDataSources lists all data sources for a knowledge base +func (s *DataSourceService) ListDataSources(ctx context.Context, kbID string) ([]*types.DataSource, error) { + dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID) + if err != nil { + logger.Errorf(ctx, "failed to list data sources: %v", err) + return nil, err + } + + // Attach latest sync log to each data source + for _, ds := range dataSources { + log, _ := s.syncLogRepo.FindLatest(ctx, ds.ID) + if log != nil { + ds.LatestSyncLog = log + } + } + + return dataSources, nil +} + +// UpdateDataSource updates an existing data source +func (s *DataSourceService) UpdateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) { + if ds == nil || ds.ID == "" { + return nil, datasource.ErrDataSourceInvalid + } + + // Verify data source exists + existing, err := s.dsRepo.FindByID(ctx, ds.ID) + if err != nil { + return nil, err + } + + // Validate new configuration if changed + if ds.Type != existing.Type || string(ds.Config) != string(existing.Config) { + if err := s.validateDataSourceConfig(ctx, ds); err != nil { + return nil, err + } + } + + if err := s.dsRepo.Update(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to update data source: %v", err) + return nil, err + } + + // Update cron schedule + if err := s.scheduler.AddOrUpdate(ds); err != nil { + logger.Warnf(ctx, "failed to update cron for ds=%s: %v", ds.ID, err) + } + + logger.Infof(ctx, "data source updated: id=%s", ds.ID) + return ds, nil +} + +// DeleteDataSource deletes a data source (soft delete) +func (s *DataSourceService) DeleteDataSource(ctx context.Context, id string) error { + // Verify data source exists + _, err := s.dsRepo.FindByID(ctx, id) + if err != nil { + return err + } + + if err := s.dsRepo.Delete(ctx, id); err != nil { + logger.Errorf(ctx, "failed to delete data source: %v", err) + return err + } + + // Remove cron schedule + s.scheduler.Remove(id) + + logger.Infof(ctx, "data source deleted: id=%s", id) + return nil +} + +// ValidateConnection tests the connection to an external data source +func (s *DataSourceService) ValidateConnection(ctx context.Context, dsID string) error { + ds, err := s.GetDataSource(ctx, dsID) + if err != nil { + return err + } + + // Get connector + connector, err := s.connectorRegistry.Get(ds.Type) + if err != nil { + return err + } + + // Parse configuration + config, err := ds.ParseConfig() + if err != nil { + return datasource.ErrInvalidConfig + } + + // Validate connection + if err := connector.Validate(ctx, config); err != nil { + // Update data source with error + ds.Status = types.DataSourceStatusError + ds.ErrorMessage = err.Error() + _ = s.dsRepo.Update(ctx, ds) + return err + } + + // Clear error if it was previously in error state + if ds.Status == types.DataSourceStatusError { + ds.Status = types.DataSourceStatusActive + ds.ErrorMessage = "" + _ = s.dsRepo.Update(ctx, ds) + } + + return nil +} + +// ListAvailableResources lists resources available for sync in the external system +func (s *DataSourceService) ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error) { + ds, err := s.GetDataSource(ctx, dsID) + if err != nil { + return nil, err + } + + // Get connector + connector, err := s.connectorRegistry.Get(ds.Type) + if err != nil { + return nil, err + } + + // Parse configuration + config, err := ds.ParseConfig() + if err != nil { + return nil, datasource.ErrInvalidConfig + } + + // List resources + resources, err := connector.ListResources(ctx, config) + if err != nil { + logger.Errorf(ctx, "failed to list resources: %v", err) + return nil, err + } + + return resources, nil +} + +// ManualSync triggers an immediate sync for a data source +func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) { + ds, err := s.GetDataSource(ctx, dsID) + if err != nil { + return nil, err + } + + if ds.Status != types.DataSourceStatusActive && ds.Status != types.DataSourceStatusError { + return nil, datasource.ErrDataSourceNotActive + } + + // Create sync log + syncLog := &types.SyncLog{ + DataSourceID: dsID, + TenantID: ds.TenantID, + Status: types.SyncLogStatusRunning, + StartedAt: time.Now().UTC(), + } + + if err := s.syncLogRepo.Create(ctx, syncLog); err != nil { + logger.Errorf(ctx, "failed to create sync log: %v", err) + return nil, err + } + + // Enqueue sync task + payload := &types.DataSourceSyncPayload{ + DataSourceID: dsID, + TenantID: ds.TenantID, + SyncLogID: syncLog.ID, + ForceFull: false, + } + + payloadJSON, _ := json.Marshal(payload) + task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON) + + _, err = s.taskEnqueuer.Enqueue(task, asynq.Queue("default")) + if err != nil { + logger.Errorf(ctx, "failed to enqueue sync task: %v", err) + // Update sync log as failed + syncLog.Status = types.SyncLogStatusFailed + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = err.Error() + _ = s.syncLogRepo.Update(ctx, syncLog) + return nil, err + } + + logger.Infof(ctx, "sync task enqueued: ds=%s syncLog=%s", dsID, syncLog.ID) + return syncLog, nil +} + +// PauseDataSource pauses a data source's scheduled syncs +func (s *DataSourceService) PauseDataSource(ctx context.Context, id string) error { + ds, err := s.GetDataSource(ctx, id) + if err != nil { + return err + } + + ds.Status = types.DataSourceStatusPaused + if err := s.dsRepo.Update(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to pause data source: %v", err) + return err + } + + // Remove cron schedule + s.scheduler.Remove(id) + + logger.Infof(ctx, "data source paused: id=%s", id) + return nil +} + +// ResumeDataSource resumes a paused data source +func (s *DataSourceService) ResumeDataSource(ctx context.Context, id string) error { + ds, err := s.GetDataSource(ctx, id) + if err != nil { + return err + } + + ds.Status = types.DataSourceStatusActive + if err := s.dsRepo.Update(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to resume data source: %v", err) + return err + } + + // Re-register cron schedule + if err := s.scheduler.AddOrUpdate(ds); err != nil { + logger.Warnf(ctx, "failed to re-register cron for ds=%s: %v", ds.ID, err) + } + + logger.Infof(ctx, "data source resumed: id=%s", id) + return nil +} + +// GetSyncLogs retrieves sync history for a data source +func (s *DataSourceService) GetSyncLogs(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) { + logs, err := s.syncLogRepo.FindByDataSource(ctx, dsID, limit, offset) + if err != nil { + logger.Errorf(ctx, "failed to get sync logs: %v", err) + return nil, err + } + return logs, nil +} + +// GetSyncLog retrieves a specific sync log entry +func (s *DataSourceService) GetSyncLog(ctx context.Context, syncLogID string) (*types.SyncLog, error) { + log, err := s.syncLogRepo.FindByID(ctx, syncLogID) + if err != nil { + return nil, err + } + return log, nil +} + +// ProcessSync handles the actual sync operation (called by asynq task) +func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) error { + var payload types.DataSourceSyncPayload + if err := json.Unmarshal(task.Payload(), &payload); err != nil { + logger.Errorf(ctx, "failed to unmarshal sync payload: %v", err) + return err + } + + logger.Infof(ctx, "processing data source sync: ds=%s syncLog=%s", payload.DataSourceID, payload.SyncLogID) + + // Get data source + ds, err := s.GetDataSource(ctx, payload.DataSourceID) + if err != nil { + logger.Errorf(ctx, "failed to get data source: %v", err) + return err + } + + // Get sync log + syncLog, err := s.syncLogRepo.FindByID(ctx, payload.SyncLogID) + if err != nil { + logger.Errorf(ctx, "failed to get sync log: %v", err) + return err + } + + // Get connector + connector, err := s.connectorRegistry.Get(ds.Type) + if err != nil { + logger.Errorf(ctx, "connector not found: type=%s", ds.Type) + syncLog.Status = types.SyncLogStatusFailed + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = fmt.Sprintf("Connector not found: %s", ds.Type) + _ = s.syncLogRepo.Update(ctx, syncLog) + return err + } + + // Parse configuration + config, err := ds.ParseConfig() + if err != nil { + logger.Errorf(ctx, "failed to parse config: %v", err) + syncLog.Status = types.SyncLogStatusFailed + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = fmt.Sprintf("Invalid configuration: %v", err) + _ = s.syncLogRepo.Update(ctx, syncLog) + return err + } + + // Fetch items based on sync mode + var items []types.FetchedItem + var nextCursor *types.SyncCursor + var fetchErr error + + if payload.ForceFull || ds.SyncMode == types.SyncModeFull { + // Full sync + items, fetchErr = connector.FetchAll(ctx, config, config.ResourceIDs) + logger.Infof(ctx, "full sync fetched %d items", len(items)) + } else { + // Incremental sync + cursor, _ := ds.ParseSyncCursor() + items, nextCursor, fetchErr = connector.FetchIncremental(ctx, config, cursor) + logger.Infof(ctx, "incremental sync fetched %d items", len(items)) + } + + if fetchErr != nil { + logger.Errorf(ctx, "fetch operation failed: %v", fetchErr) + syncLog.Status = types.SyncLogStatusFailed + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = fmt.Sprintf("Fetch failed: %v", fetchErr) + _ = s.syncLogRepo.Update(ctx, syncLog) + return fetchErr + } + + // Process fetched items and write to knowledge base + var result = &types.SyncResult{ + Total: len(items), + } + + // Set tenant context so KnowledgeService can resolve tenant info correctly + ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID) + + tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID) + if err != nil { + logger.Errorf(ctx, "failed to get tenant info: %v", err) + syncLog.Status = types.SyncLogStatusFailed + syncLog.FinishedAt = timePtr(time.Now()) + syncLog.ErrorMessage = fmt.Sprintf("Failed to get tenant info: %v", err) + _ = s.syncLogRepo.Update(ctx, syncLog) + return err + } + ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) + + // Auto-tag: find or create a tag for this data source so synced items are easily identifiable + autoTagID := "" + autoTagName := connectorDisplayLabel(ds.Type) + "·" + ds.Name + if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, autoTagName); tagErr != nil { + logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", autoTagName, tagErr) + } else if autoTag != nil { + autoTagID = autoTag.ID + logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", autoTagName, autoTagID) + } + + for _, item := range items { + if item.IsDeleted { + if ds.SyncDeletions { + result.Deleted++ + } + continue + } + + if len(item.Content) == 0 && item.URL == "" { + // Check if this is an error item from the connector (failed to fetch content) + if errMsg, hasErr := item.Metadata["error"]; hasErr { + logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg) + result.Failed++ + result.Errors = append(result.Errors, fmt.Sprintf("%s: %s", item.Title, errMsg)) + } else { + logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID) + result.Skipped++ + } + continue + } + + isUpdate, err := s.ingestItem(ctx, ds, &item, autoTagID) + if err != nil { + // Duplicate file/URL is not a failure — count as skipped + var dupErr *types.DuplicateKnowledgeError + if errors.As(err, &dupErr) { + logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID) + result.Skipped++ + } else { + logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err) + result.Failed++ + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", item.Title, err)) + } + } else if isUpdate { + result.Updated++ + } else { + result.Created++ + } + } + + // Update sync log with results + syncLog.ItemsTotal = result.Total + syncLog.ItemsCreated = result.Created + syncLog.ItemsUpdated = result.Updated + syncLog.ItemsDeleted = result.Deleted + syncLog.ItemsSkipped = result.Skipped + syncLog.ItemsFailed = result.Failed + syncLog.Status = types.SyncLogStatusSuccess + syncLog.FinishedAt = timePtr(time.Now()) + + // Update cursor for next incremental sync + if nextCursor != nil { + cursorJSON, _ := nextCursor.ToJSON() + ds.LastSyncCursor = cursorJSON + } + + ds.LastSyncAt = timePtr(time.Now()) + ds.Status = types.DataSourceStatusActive + ds.ErrorMessage = "" + + // Store result + resultJSON, _ := result.ToJSON() + ds.LastSyncResult = resultJSON + syncLog.Result = resultJSON + + // Update database + if err := s.dsRepo.Update(ctx, ds); err != nil { + logger.Errorf(ctx, "failed to update data source: %v", err) + } + if err := s.syncLogRepo.Update(ctx, syncLog); err != nil { + logger.Errorf(ctx, "failed to update sync log: %v", err) + } + + logger.Infof(ctx, "data source sync completed: ds=%s created=%d updated=%d deleted=%d", + payload.DataSourceID, syncLog.ItemsCreated, syncLog.ItemsUpdated, syncLog.ItemsDeleted) + + return nil +} + +// ValidateCredentials tests connectivity using raw credentials without persisting anything. +func (s *DataSourceService) ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error { + connector, err := s.connectorRegistry.Get(connectorType) + if err != nil { + return err + } + + config := &types.DataSourceConfig{ + Type: connectorType, + Credentials: credentials, + } + + if err := connector.Validate(ctx, config); err != nil { + return err + } + + return nil +} + +// Helper functions + +func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *types.DataSource) error { + connector, err := s.connectorRegistry.Get(ds.Type) + if err != nil { + return err + } + + config, err := ds.ParseConfig() + if err != nil { + return datasource.ErrInvalidConfig + } + + return connector.Validate(ctx, config) +} + +// ingestItem writes a single FetchedItem into the knowledge base. +// If a knowledge item with the same external_id already exists, it is deleted first (update = delete + re-create). +// +// Routing logic: +// - Has Content bytes → CreateKnowledgeFromFile (走完整的文档解析 pipeline) +// - Has URL only → CreateKnowledgeFromURL (让 WeKnora 下载并解析) +// +// Returns (isUpdate, error) — isUpdate is true when an existing item was replaced. +func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource, item *types.FetchedItem, tagID string) (bool, error) { + channel := ds.Type // e.g. "feishu", "notion" + + metadata := map[string]string{ + "external_id": item.ExternalID, + "source_resource_id": item.SourceResourceID, + "datasource_id": ds.ID, + } + for k, v := range item.Metadata { + metadata[k] = v + } + + // Check if a knowledge item with this external_id already exists → delete it first (update) + isUpdate := false + if item.ExternalID != "" { + repo := s.knowledgeService.GetRepository() + existing, err := repo.FindByMetadataKey(ctx, ds.TenantID, ds.KnowledgeBaseID, "external_id", item.ExternalID) + if err != nil { + logger.Warnf(ctx, "failed to check existing knowledge for external_id=%s: %v", item.ExternalID, err) + // Non-fatal: proceed with creation (may produce duplicate) + } else if existing != nil { + logger.Infof(ctx, "found existing knowledge %s for external_id=%s, deleting for update", existing.ID, item.ExternalID) + if err := s.knowledgeService.DeleteKnowledge(ctx, existing.ID); err != nil { + logger.Warnf(ctx, "failed to delete existing knowledge %s: %v", existing.ID, err) + } else { + isUpdate = true + } + } + } + + // Case 1: content already fetched → build a FileHeader from bytes and call CreateKnowledgeFromFile + if len(item.Content) > 0 { + fh, err := bytesToFileHeader(item.Content, item.FileName) + if err != nil { + return isUpdate, fmt.Errorf("build file header: %w", err) + } + _, err = s.knowledgeService.CreateKnowledgeFromFile( + ctx, + ds.KnowledgeBaseID, + fh, + metadata, + nil, // use KB default for multimodal + item.FileName, // customFileName — must include extension for file-type validation + tagID, // auto-tag from data source + channel, + ) + return isUpdate, err + } + + // Case 2: only a remote URL — let WeKnora handle downloading and parsing + if item.URL != "" { + _, err := s.knowledgeService.CreateKnowledgeFromURL( + ctx, + ds.KnowledgeBaseID, + item.URL, + item.FileName, + "", // auto-detect file type + nil, // use KB default for multimodal + item.Title, + tagID, // auto-tag from data source + channel, + ) + return isUpdate, err + } + + return isUpdate, fmt.Errorf("item has neither content nor URL") +} + +// bytesToFileHeader wraps a []byte into a *multipart.FileHeader so it can be +// consumed by KnowledgeService.CreateKnowledgeFromFile. +func bytesToFileHeader(data []byte, filename string) (*multipart.FileHeader, error) { + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Create a form file part + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename)) + partHeader.Set("Content-Type", "application/octet-stream") + + part, err := writer.CreatePart(partHeader) + if err != nil { + return nil, fmt.Errorf("create multipart part: %w", err) + } + + if _, err := part.Write(data); err != nil { + return nil, fmt.Errorf("write data to part: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("close multipart writer: %w", err) + } + + // Parse the multipart data to get a FileHeader + reader := multipart.NewReader(&buf, writer.Boundary()) + form, err := reader.ReadForm(int64(len(data)) + 1024) + if err != nil { + return nil, fmt.Errorf("read multipart form: %w", err) + } + + files := form.File["file"] + if len(files) == 0 { + return nil, fmt.Errorf("no file in multipart form") + } + + return files[0], nil +} + +func timePtr(t time.Time) *time.Time { + utc := t.UTC() + return &utc +} + +// connectorDisplayLabel returns a human-readable label for a connector type. +// Used to generate auto-tag names like "飞书·我的知识库". +func connectorDisplayLabel(connectorType string) string { + labels := map[string]string{ + "feishu": "飞书", + "notion": "Notion", + "confluence": "Confluence", + "yuque": "语雀", + "github": "GitHub", + "google_drive": "Google Drive", + "onedrive": "OneDrive", + "dingtalk": "钉钉", + "wecom_doc": "企微文档", + "web_crawler": "网页爬虫", + "slack": "Slack", + "imap": "邮件", + "rss": "RSS", + } + if l, ok := labels[connectorType]; ok { + return l + } + return connectorType +} diff --git a/internal/container/container.go b/internal/container/container.go index cc9281a9..9c34adff 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -50,6 +50,8 @@ import ( "github.com/Tencent/WeKnora/internal/application/service/web_search" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/database" + "github.com/Tencent/WeKnora/internal/datasource" + feishuConnector "github.com/Tencent/WeKnora/internal/datasource/connector/feishu" "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/handler" "github.com/Tencent/WeKnora/internal/handler/session" @@ -143,6 +145,8 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(repository.NewAgentShareRepository)) must(container.Provide(repository.NewTenantDisabledSharedAgentRepository)) must(container.Provide(service.NewWebSearchStateService)) + must(container.Provide(repository.NewDataSourceRepository)) + must(container.Provide(repository.NewSyncLogRepository)) // MCP manager for managing MCP client connections logger.Debugf(ctx, "[Container] Registering MCP manager...") @@ -204,6 +208,14 @@ func BuildContainer(container *dig.Container) *dig.Container { // Chat pipeline components for processing chat requests logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...") + + // Data source sync framework + logger.Debugf(ctx, "[Container] Registering data source sync framework...") + must(container.Provide(initConnectorRegistry)) + must(container.Provide(datasource.NewScheduler)) + must(container.Provide(service.NewDataSourceService)) + must(container.Invoke(startDataSourceScheduler)) + logger.Debugf(ctx, "[Container] Data source sync framework registered") must(container.Provide(chatpipeline.NewEventManager)) must(container.Invoke(chatpipeline.NewPluginSearch)) must(container.Invoke(chatpipeline.NewPluginRerank)) @@ -243,6 +255,9 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(handler.NewSkillHandler)) must(container.Provide(handler.NewOrganizationHandler)) + // Data source handler + must(container.Provide(handler.NewDataSourceHandler)) + // IM integration logger.Debugf(ctx, "[Container] Registering IM integration...") must(container.Provide(imPkg.NewService)) @@ -397,7 +412,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { default: return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER")) } - db, err := gorm.Open(dialector, &gorm.Config{}) + db, err := gorm.Open(dialector, &gorm.Config{ + NowFunc: func() time.Time { + return time.Now().UTC() + }, + }) if err != nil { return nil, err } @@ -1254,3 +1273,31 @@ func credentialBool(creds map[string]interface{}, key string) bool { return false } } + +// initConnectorRegistry creates and populates the connector registry with all available connectors. +func initConnectorRegistry() *datasource.ConnectorRegistry { + registry := datasource.NewConnectorRegistry() + + // Register Feishu connector + _ = registry.Register(feishuConnector.NewConnector()) + + // Future connectors will be registered here: + // _ = registry.Register(notionConnector.NewConnector()) + // _ = registry.Register(confluenceConnector.NewConnector()) + // _ = registry.Register(yuqueConnector.NewConnector()) + // _ = registry.Register(githubConnector.NewConnector()) + + return registry +} + +// startDataSourceScheduler starts the data source cron scheduler and registers cleanup. +func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interfaces.ResourceCleaner) { + if err := scheduler.Start(context.Background()); err != nil { + logger.Warnf(context.Background(), "[Container] data source scheduler start failed: %v", err) + } + + cleaner.RegisterWithName("DataSourceScheduler", func() error { + scheduler.Stop() + return nil + }) +} diff --git a/internal/datasource/CONNECTOR_IMPLEMENTATION_GUIDE.md b/internal/datasource/CONNECTOR_IMPLEMENTATION_GUIDE.md new file mode 100644 index 00000000..72772e46 --- /dev/null +++ b/internal/datasource/CONNECTOR_IMPLEMENTATION_GUIDE.md @@ -0,0 +1,497 @@ +# Adding a New Connector - Step-by-Step Guide + +This guide walks you through implementing a new connector (e.g., Feishu, Notion, Confluence) for the data source sync framework. + +## Overview + +A connector is an adapter that translates between WeKnora's data model and an external platform's API. It handles: +- Connection validation (credentials, connectivity) +- Resource listing (documents, spaces, folders the user can choose from) +- Full sync (fetch all items from selected resources) +- Incremental sync (fetch only changed items since last sync) + +## Step 1: Create Connector Package Structure + +```bash +mkdir -p internal/datasource/connector/yourtype/ +``` + +Create three files: +- `client.go` - API client wrapper +- `connector.go` - Implements Connector interface +- `types.go` - Platform-specific data structures + +## Step 2: Define Platform Types (types.go) + +```go +package yourtype + +import "time" + +// Platform-specific configuration +type Config struct { + BaseURL string `json:"base_url"` + APIToken string `json:"api_token"` + + // Or OAuth fields: + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresAt time.Time `json:"expires_at"` +} + +// Platform-specific resource representation +type YourResource struct { + ID string + Name string + Type string // "document", "folder", "space", etc. + ModifiedAt time.Time + URL string +} + +// Platform-specific item representation +type YourItem struct { + ID string + Title string + Content string + ContentHTML string + ModifiedAt time.Time + URL string + CreatedBy string +} + +// Platform-specific pagination/cursor +type YourCursor struct { + Offset int `json:"offset,omitempty"` + LastModified time.Time `json:"last_modified,omitempty"` + PageToken string `json:"page_token,omitempty"` +} +``` + +## Step 3: Implement API Client (client.go) + +```go +package yourtype + +import ( + "context" + "fmt" + "net/http" + "encoding/json" +) + +type Client struct { + baseURL string + apiToken string + httpClient *http.Client +} + +// NewClient creates a new API client +func NewClient(config *Config) *Client { + return &Client{ + baseURL: config.BaseURL, + apiToken: config.APIToken, + httpClient: &http.Client{Timeout: 30 * time.Second}, + } +} + +// Example methods +func (c *Client) GetResources(ctx context.Context) ([]YourResource, error) { + // Call platform API + // Parse response + // Return resources +} + +func (c *Client) GetDocument(ctx context.Context, docID string) (*YourItem, error) { + // Fetch single document +} + +func (c *Client) GetDocumentsModifiedSince(ctx context.Context, since time.Time) ([]YourItem, error) { + // Fetch documents modified since timestamp +} +``` + +## Step 4: Implement Connector Interface (connector.go) + +```go +package yourtype + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +type YourConnector struct { + client *Client +} + +// NewConnector creates a new connector +func NewConnector() *YourConnector { + return &YourConnector{} +} + +// Type returns the connector type identifier +func (c *YourConnector) Type() string { + return types.ConnectorTypeYourType // Must match constant in types/datasource.go +} + +// Validate verifies that the configuration is valid +func (c *YourConnector) Validate(ctx context.Context, config *types.DataSourceConfig) error { + if config == nil { + return fmt.Errorf("config is nil") + } + + // Parse your type-specific config + yourConfig := &Config{} + if err := parseConfig(config, yourConfig); err != nil { + return fmt.Errorf("invalid config: %w", err) + } + + // Create client + client := NewClient(yourConfig) + + // Test connection + _, err := client.GetResources(ctx) + if err != nil { + return fmt.Errorf("connection failed: %w", err) + } + + return nil +} + +// ListResources lists available resources (documents, spaces, folders) +func (c *YourConnector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) { + yourConfig := &Config{} + if err := parseConfig(config, yourConfig); err != nil { + return nil, err + } + + client := NewClient(yourConfig) + yourResources, err := client.GetResources(ctx) + if err != nil { + return nil, err + } + + // Convert to WeKnora Resource format + resources := make([]types.Resource, len(yourResources)) + for i, yr := range yourResources { + resources[i] = types.Resource{ + ExternalID: yr.ID, + Name: yr.Name, + Type: yr.Type, + URL: yr.URL, + ModifiedAt: yr.ModifiedAt, + } + } + + return resources, nil +} + +// FetchAll performs a full sync +func (c *YourConnector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) { + yourConfig := &Config{} + if err := parseConfig(config, yourConfig); err != nil { + return nil, err + } + + client := NewClient(yourConfig) + + var allItems []types.FetchedItem + + // Fetch all documents from specified resources + for _, resourceID := range resourceIDs { + // Get documents from this resource (implementation depends on platform) + yourItems, err := client.GetDocumentsFromResource(ctx, resourceID) + if err != nil { + return nil, fmt.Errorf("failed to fetch resource %s: %w", resourceID, err) + } + + // Convert to FetchedItem format + for _, yi := range yourItems { + item := types.FetchedItem{ + ExternalID: yi.ID, + Title: yi.Title, + Content: []byte(yi.Content), + ContentType: "text/markdown", + FileName: fmt.Sprintf("%s.md", yi.Title), + URL: yi.URL, + UpdatedAt: yi.ModifiedAt, + SourceResourceID: resourceID, + Metadata: map[string]string{ + "created_by": yi.CreatedBy, + "platform": "yourtype", + }, + } + allItems = append(allItems, item) + } + } + + return allItems, nil +} + +// FetchIncremental performs an incremental sync +func (c *YourConnector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) { + yourConfig := &Config{} + if err := parseConfig(config, yourConfig); err != nil { + return nil, nil, err + } + + client := NewClient(yourConfig) + + // Determine start time for incremental fetch + var sinceTime time.Time + if cursor != nil && !cursor.LastSyncTime.IsZero() { + sinceTime = cursor.LastSyncTime + } else { + sinceTime = time.Now().AddDate(0, 0, -7) // Default: last 7 days + } + + // Fetch changed items + yourItems, err := client.GetDocumentsModifiedSince(ctx, sinceTime) + if err != nil { + return nil, nil, fmt.Errorf("incremental fetch failed: %w", err) + } + + // Convert to FetchedItem format + items := make([]types.FetchedItem, len(yourItems)) + for i, yi := range yourItems { + items[i] = types.FetchedItem{ + ExternalID: yi.ID, + Title: yi.Title, + Content: []byte(yi.Content), + ContentType: "text/markdown", + FileName: fmt.Sprintf("%s.md", yi.Title), + URL: yi.URL, + UpdatedAt: yi.ModifiedAt, + Metadata: map[string]string{ + "created_by": yi.CreatedBy, + "platform": "yourtype", + }, + } + } + + // Create new cursor for next sync + nextCursor := &types.SyncCursor{ + LastSyncTime: time.Now(), + ConnectorCursor: map[string]interface{}{ + "last_modified": time.Now(), + }, + } + + return items, nextCursor, nil +} + +// Helper function to parse config +func parseConfig(config *types.DataSourceConfig, target interface{}) error { + data, err := json.Marshal(config.Credentials) + if err != nil { + return err + } + return json.Unmarshal(data, target) +} +``` + +## Step 5: Register in Container + +Edit `internal/container/container.go`: + +```go +import ( + // ... existing imports + yourconnector "github.com/Tencent/WeKnora/internal/datasource/connector/yourtype" +) + +func setupContainer() (*dig.Container, error) { + container := dig.New() + + // ... existing registrations ... + + // Register your connector + must(container.Provide(func() *datasource.Connector { + var c datasource.Connector = yourconnector.NewConnector() + return &c + })) + + // ... rest of setup ... +} +``` + +Better yet, register through the registry in the connector setup: + +```go +// In the service initialization section +connectorRegistry := datasource.NewConnectorRegistry() +connectorRegistry.Register(yourconnector.NewConnector()) +connectorRegistry.Register(feishuconnector.NewConnector()) +// ... etc + +container.Provide(func() *datasource.ConnectorRegistry { + return connectorRegistry +}) +``` + +## Step 6: Add Connector Type Constant + +Edit `internal/types/datasource.go`: + +```go +const ( + // ... existing types ... + ConnectorTypeYourType = "yourtype" +) +``` + +## Step 7: Add Metadata + +Edit `internal/datasource/connector.go`: + +```go +var ConnectorMetadataRegistry = map[string]ConnectorMetadata{ + // ... existing entries ... + types.ConnectorTypeYourType: { + Type: types.ConnectorTypeYourType, + Name: "Your Platform Name", + Description: "Sync documents from Your Platform", + Priority: X, // Lower number = higher priority in UI + AuthType: "oauth2", // or "api_key", "token", "password" + Capabilities: []string{"incremental", "webhook", "deletion_sync"}, + }, +} +``` + +## Step 8: Test Your Connector + +```go +// Example test +func TestYourConnectorValidate(t *testing.T) { + connector := NewConnector() + + config := &types.DataSourceConfig{ + Type: types.ConnectorTypeYourType, + Credentials: map[string]interface{}{ + "api_token": "test_token", + }, + } + + err := connector.Validate(context.Background(), config) + // assert no error +} + +func TestYourConnectorFetchAll(t *testing.T) { + connector := NewConnector() + config := &types.DataSourceConfig{ + Type: types.ConnectorTypeYourType, + Credentials: map[string]interface{}{ + "api_token": "test_token", + }, + ResourceIDs: []string{"resource_1"}, + } + + items, err := connector.FetchAll(context.Background(), config, []string{"resource_1"}) + // assert results +} +``` + +## Checklist + +- [ ] Created `internal/datasource/connector/yourtype/` package +- [ ] Implemented `types.go` with platform data structures +- [ ] Implemented `client.go` with API wrapper +- [ ] Implemented `connector.go` with Connector interface +- [ ] Added connector type constant +- [ ] Registered in container +- [ ] Added metadata entry +- [ ] Added unit tests +- [ ] Tested manually with real API +- [ ] Documented any special requirements + +## Common Patterns + +### OAuth Flow +If using OAuth, store tokens in config: +```go +type Config struct { + AccessToken string + RefreshToken string + ExpiresAt time.Time +} + +// Refresh tokens when expired +func (c *Client) ensureValidToken(ctx context.Context) error { + if time.Now().After(c.config.ExpiresAt) { + return c.refreshToken(ctx) + } + return nil +} +``` + +### Pagination +For platforms with pagination: +```go +func (c *Client) GetDocumentsPage(ctx context.Context, pageToken string) (*Page, error) { + // Returns {Items, NextPageToken} +} +``` + +### Incremental Sync +For timestamp-based incremental sync: +```go +func (c *Client) GetModifiedSince(ctx context.Context, since time.Time) ([]Item, error) { + // Uses API parameter like &modified_after=2026-03-26T10:00:00Z +} +``` + +### Deletion Tracking +For platforms that report deletions: +```go +type Item struct { + IsDeleted bool // Set when item is deleted +} +``` + +## Testing with Real API + +1. Set up test credentials +2. Create small test resource (e.g., single document) +3. Test each method: + ```go + connector := NewConnector() + config := &types.DataSourceConfig{...} + + // Test Validate + err := connector.Validate(ctx, config) + + // Test ListResources + resources, err := connector.ListResources(ctx, config) + + // Test FetchAll + items, err := connector.FetchAll(ctx, config, []string{resources[0].ExternalID}) + + // Test FetchIncremental + items, cursor, err := connector.FetchIncremental(ctx, config, nil) + ``` + +--- + +## Example: Feishu Connector Reference + +The Feishu connector would be a good first implementation since: +1. Feishu API is well-documented +2. WeKnora already has `internal/im/feishu/` for reference +3. Popular in China (key market) +4. Has webhook support for real-time sync + +Structure: +``` +internal/datasource/connector/feishu/ +├── client.go (Feishu API client) +├── connector.go (Implements Connector) +└── types.go (Feishu types) +``` + +Key files to reference: +- `internal/im/feishu/adapter.go` - Feishu API patterns +- `internal/im/feishu/longconn.go` - Connection handling + +This will be a good model for other connectors. diff --git a/internal/datasource/README.md b/internal/datasource/README.md new file mode 100644 index 00000000..9512515d --- /dev/null +++ b/internal/datasource/README.md @@ -0,0 +1,260 @@ +# WeKnora Data Source Sync Framework + +## Overview + +The data source sync framework enables WeKnora to automatically import and synchronize content from external platforms (Feishu, Notion, Confluence, etc.) into knowledge bases. This is the foundational layer upon which all specific connectors are built. + +## Architecture + +### Core Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ External Data Sources │ +│ (Feishu, Notion, Confluence, GitHub, Google Drive...) │ +└─────────────┬───────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Connector Registry & Adapters │ +│ Each platform (feishu/, notion/, confluence/, etc) │ +│ implements the Connector interface │ +└─────────────┬───────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ DataSourceService (Business Logic) │ +│ ├─ CreateDataSource / UpdateDataSource │ +│ ├─ ManualSync / ListAvailableResources │ +│ ├─ ValidateConnection / PauseDataSource │ +│ └─ ProcessSync (asynq task handler) │ +└─────────────┬───────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ HTTP Handler & API Routes (/api/v1/datasource) │ +│ REST endpoints for UI/programmatic access │ +└─────────────┬───────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Database (MySQL / PostgreSQL) │ +│ ├─ data_sources (configuration & state) │ +│ └─ sync_logs (history & metadata) │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Task Flow + +``` +Manual Trigger or Scheduled Job (cron) + ↓ + Scheduler enqueues asynq Task + ↓ + ProcessSync (task handler) executes + ↓ + Connector.Fetch* methods pull data + ↓ + Diff engine identifies changes (new/update/delete) + ↓ + KnowledgeService creates/updates Knowledge entries + ↓ + Documents parsed → chunks → vectors → indexed +``` + +## File Structure + +``` +internal/datasource/ +├── connector.go # Connector interface & registry +├── errors.go # Error definitions +└── README.md # This file + +internal/types/ +├── datasource.go # Data models (DataSource, SyncLog, etc) +└── interfaces/datasource.go # Service interfaces + +internal/application/ +├── repository/datasource_repo.go # Database access layer +└── service/datasource_service.go # Business logic + +internal/handler/ +└── datasource.go # HTTP request handlers + +internal/router/ +├── router.go # Route registration +├── task.go # Asynq task registration +└── sync_task.go # Lite mode task registration + +migrations/versioned/ +└── 000028_datasource_tables.up/down.sql # Database schema +``` + +## Data Models + +### DataSource +Represents a configured external data source: +- **ID**: Unique identifier +- **Type**: Connector type (feishu, notion, confluence, etc) +- **Config**: Encrypted credentials and configuration (JSON) +- **SyncSchedule**: Cron expression (e.g., "0 */6 * * *") +- **SyncMode**: "incremental" or "full" +- **Status**: active | paused | error +- **ConflictStrategy**: overwrite | skip +- **SyncDeletions**: Whether to sync deletions from source +- **LastSyncAt**: Timestamp of last successful sync +- **LastSyncCursor**: State for incremental sync +- **LastSyncResult**: Summary of last sync + +### SyncLog +Tracks execution of each sync operation: +- **Status**: running | success | partial | failed | canceled +- **StartedAt/FinishedAt**: Execution timestamps +- **Items***: Counters (total, created, updated, deleted, skipped, failed) +- **ErrorMessage**: Error details if failed +- **Result**: Detailed sync result (JSON) + +### Connector Interface +All external data sources implement this interface: + +```go +type Connector interface { + Type() string + Validate(ctx, config) error + ListResources(ctx, config) ([]Resource, error) + FetchAll(ctx, config, resourceIDs) ([]FetchedItem, error) + FetchIncremental(ctx, config, cursor) ([]FetchedItem, *SyncCursor, error) +} +``` + +## API Endpoints + +### Data Source Management +``` +POST /api/v1/datasource # Create +GET /api/v1/datasource # List (by kb_id) +GET /api/v1/datasource/:id # Get +PUT /api/v1/datasource/:id # Update +DELETE /api/v1/datasource/:id # Delete +``` + +### Operations +``` +POST /api/v1/datasource/:id/validate # Test connection +GET /api/v1/datasource/:id/resources # List available resources +POST /api/v1/datasource/:id/sync # Trigger manual sync +POST /api/v1/datasource/:id/pause # Pause +POST /api/v1/datasource/:id/resume # Resume +``` + +### Logs +``` +GET /api/v1/datasource/:id/logs # List sync logs +GET /api/v1/datasource/logs/:log_id # Get specific log +``` + +### Metadata +``` +GET /api/v1/datasource/types # Available connectors +``` + +## Implementation Guide + +### Adding a New Connector + +1. **Create the connector package** in `internal/datasource/connector//` + ```go + // connector//client.go - API client + // connector//connector.go - Implements Connector interface + // connector//types.go - Type-specific models + ``` + +2. **Implement the Connector interface** + ```go + type YourConnector struct { + // fields + } + + func (c *YourConnector) Type() string { + return types.ConnectorTypeYourType + } + + func (c *YourConnector) Validate(ctx, config) error { + // Test connectivity and credentials + } + + func (c *YourConnector) ListResources(ctx, config) ([]types.Resource, error) { + // List available resources (documents, spaces, etc) + } + + func (c *YourConnector) FetchAll(ctx, config, resourceIDs) ([]types.FetchedItem, error) { + // Full sync - fetch all items from specified resources + } + + func (c *YourConnector) FetchIncremental(ctx, config, cursor) ([]types.FetchedItem, *types.SyncCursor, error) { + // Incremental sync - fetch only changed items since cursor + } + ``` + +3. **Register the connector** in the container (in container.go) + ```go + container.Provide(func() datasource.Connector { + return yourconnector.NewConnector() + }) + ``` + +4. **Add metadata** in `connector.go` + ```go + types.ConnectorTypeYourType: { + Type: types.ConnectorTypeYourType, + Name: "Your Platform", + AuthType: "oauth2", + // ... + } + ``` + +## Database Schema + +### data_sources table +- Stores data source configurations +- Indexes: tenant_id, knowledge_base_id, type, status +- Encrypted config field (AES-256-GCM) + +### sync_logs table +- Stores sync operation history +- Indexes: data_source_id, tenant_id, status, started_at +- Foreign key to data_sources with cascade delete + +## Key Design Decisions + +1. **Adapter Pattern**: Each connector is a separate implementation, easy to add new ones +2. **Async Task Queue**: Syncs are non-blocking, using asynq (or sync mode in Lite) +3. **Incremental Sync**: Supports cursor-based incremental syncs to minimize API calls +4. **Encryption**: API keys/tokens stored encrypted with AES-256-GCM +5. **Multi-tenant**: All operations are tenant-isolated +6. **Channel Tracking**: Knowledge.Channel field tracks data source type for origin tracking + +## Future Enhancements + +1. **Webhook Support**: Real-time push from platforms that support webhooks +2. **Conflict Resolution**: Advanced merge/conflict strategies for overlapping content +3. **Rate Limiting**: Per-connector rate limiting and backoff strategies +4. **Scheduling**: Full cron scheduler with time zone support +5. **Monitoring**: Metrics, alerting, and sync health dashboards +6. **Filtering**: User-defined filters for selective syncing (by title, date, tags, etc) +7. **Transformation**: Content transformation pipelines (e.g., extract tables, summarize) +8. **Deduplication**: Smart deduplication across multiple sources + +## Testing + +The framework is designed to be testable: +- Mock connectors can be created for testing +- No external dependencies required for core logic +- Database operations isolated and mockable + +## Error Handling + +All operations return detailed error information: +- Connection failures are captured and status updated +- Partial failures are logged (some items succeed, some fail) +- Error messages stored in DataSource for user troubleshooting diff --git a/internal/datasource/connector.go b/internal/datasource/connector.go new file mode 100644 index 00000000..29cf1492 --- /dev/null +++ b/internal/datasource/connector.go @@ -0,0 +1,221 @@ +package datasource + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Connector is the interface that all external data source connectors must implement. +// Each connector (Feishu, Notion, Confluence, etc.) provides an implementation of this interface. +type Connector interface { + // Type returns the connector type identifier (e.g., "feishu", "notion") + Type() string + + // Validate verifies that the provided configuration is valid by testing connectivity + // and checking credentials. Returns error if validation fails. + Validate(ctx context.Context, config *types.DataSourceConfig) error + + // ListResources lists available resources that can be synced (documents, spaces, folders, etc.) + // Returns a list of Resource objects that the user can select for syncing. + ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) + + // FetchAll performs a full sync of the specified resources. + // Returns all items from the given resource IDs. + FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) + + // FetchIncremental performs an incremental sync based on the provided cursor. + // Returns items that have changed since the last sync, a new cursor for the next sync, + // and an error if the operation fails. + FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) +} + +// ConnectorRegistry manages the registration and lookup of available connectors +type ConnectorRegistry struct { + connectors map[string]Connector +} + +// NewConnectorRegistry creates a new connector registry +func NewConnectorRegistry() *ConnectorRegistry { + return &ConnectorRegistry{ + connectors: make(map[string]Connector), + } +} + +// Register registers a connector with the registry +func (r *ConnectorRegistry) Register(connector Connector) error { + if connector == nil { + return ErrConnectorNil + } + if connector.Type() == "" { + return ErrConnectorTypeEmpty + } + r.connectors[connector.Type()] = connector + return nil +} + +// Get retrieves a connector by type +func (r *ConnectorRegistry) Get(connectorType string) (Connector, error) { + connector, exists := r.connectors[connectorType] + if !exists { + return nil, ErrConnectorNotFound + } + return connector, nil +} + +// List returns all registered connector types +func (r *ConnectorRegistry) List() []string { + types := make([]string, 0, len(r.connectors)) + for t := range r.connectors { + types = append(types, t) + } + return types +} + +// ConnectorMetadata provides metadata about available connectors +type ConnectorMetadata struct { + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Icon string `json:"icon,omitempty"` + Priority int `json:"priority"` // Priority order for UI display (lower = higher priority) + AuthType string `json:"auth_type"` // "oauth2", "api_key", "token", etc. + Capabilities []string `json:"capabilities"` // "incremental", "webhook", "deletion_sync", etc. +} + +// GetConnectorMetadata returns metadata for all available connectors +// This is used by the frontend to display connector options +var ConnectorMetadataRegistry = map[string]ConnectorMetadata{ + types.ConnectorTypeFeishu: { + Type: types.ConnectorTypeFeishu, + Name: "Feishu (飞书)", + Description: "Sync documents, wikis, and content from Feishu", + Priority: 0, + AuthType: "oauth2", + Capabilities: []string{"incremental", "webhook", "deletion_sync"}, + }, + types.ConnectorTypeNotion: { + Type: types.ConnectorTypeNotion, + Name: "Notion", + Description: "Sync pages and databases from Notion", + Priority: 1, + AuthType: "api_key", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeConfluence: { + Type: types.ConnectorTypeConfluence, + Name: "Confluence", + Description: "Sync spaces and pages from Atlassian Confluence", + Priority: 2, + AuthType: "api_key", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeYuque: { + Type: types.ConnectorTypeYuque, + Name: "Yuque (语雀)", + Description: "Sync knowledge bases and documents from Yuque", + Priority: 3, + AuthType: "api_key", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeGitHub: { + Type: types.ConnectorTypeGitHub, + Name: "GitHub", + Description: "Sync repositories, wikis, and issues from GitHub", + Priority: 4, + AuthType: "oauth2", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeGoogleDrive: { + Type: types.ConnectorTypeGoogleDrive, + Name: "Google Drive", + Description: "Sync documents and files from Google Drive", + Priority: 5, + AuthType: "oauth2", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeOneDrive: { + Type: types.ConnectorTypeOneDrive, + Name: "OneDrive / SharePoint", + Description: "Sync documents and files from Microsoft OneDrive", + Priority: 6, + AuthType: "oauth2", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeDingTalk: { + Type: types.ConnectorTypeDingTalk, + Name: "DingTalk (钉钉)", + Description: "Sync documents and content from DingTalk", + Priority: 7, + AuthType: "api_key", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeWecomDoc: { + Type: types.ConnectorTypeWecomDoc, + Name: "WeCom Docs (企微文档)", + Description: "Sync documents and content from WeCom", + Priority: 8, + AuthType: "api_key", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeWebCrawler: { + Type: types.ConnectorTypeWebCrawler, + Name: "Web Crawler (Sitemap)", + Description: "Crawl websites via Sitemap.xml", + Priority: 9, + AuthType: "none", + Capabilities: []string{}, + }, + types.ConnectorTypeSlack: { + Type: types.ConnectorTypeSlack, + Name: "Slack", + Description: "Sync channel messages and files from Slack", + Priority: 10, + AuthType: "oauth2", + Capabilities: []string{"incremental"}, + }, + types.ConnectorTypeIMAP: { + Type: types.ConnectorTypeIMAP, + Name: "Email (IMAP)", + Description: "Sync email content from IMAP servers", + Priority: 11, + AuthType: "password", + Capabilities: []string{}, + }, + types.ConnectorTypeRSS: { + Type: types.ConnectorTypeRSS, + Name: "RSS / Atom Feed", + Description: "Sync articles from RSS/Atom feeds", + Priority: 12, + AuthType: "none", + Capabilities: []string{}, + }, +} + +// GetConnectorMetadata returns metadata for a specific connector type +func GetConnectorMetadata(connectorType string) (ConnectorMetadata, bool) { + meta, ok := ConnectorMetadataRegistry[connectorType] + return meta, ok +} + +// ListAvailableConnectors returns all available connector metadata +// sorted by priority +func ListAvailableConnectors() []ConnectorMetadata { + metadata := make([]ConnectorMetadata, 0, len(ConnectorMetadataRegistry)) + for _, meta := range ConnectorMetadataRegistry { + metadata = append(metadata, meta) + } + + // Sort by priority (insertion sort for simplicity) + for i := 1; i < len(metadata); i++ { + key := metadata[i] + j := i - 1 + for j >= 0 && metadata[j].Priority > key.Priority { + metadata[j+1] = metadata[j] + j-- + } + metadata[j+1] = key + } + + return metadata +} diff --git a/internal/datasource/connector/feishu/client.go b/internal/datasource/connector/feishu/client.go new file mode 100644 index 00000000..cb159880 --- /dev/null +++ b/internal/datasource/connector/feishu/client.go @@ -0,0 +1,451 @@ +package feishu + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/logger" +) + +// Client wraps the Feishu Open Platform API for document/wiki operations. +type Client struct { + baseURL string + appID string + appSecret string + + httpClient *http.Client + + // Token cache (thread-safe) + tokenMu sync.Mutex + tokenCache string + tokenExpAt time.Time +} + +// NewClient creates a new Feishu API client. +func NewClient(config *Config) *Client { + return &Client{ + baseURL: config.GetBaseURL(), + appID: config.AppID, + appSecret: config.AppSecret, + httpClient: &http.Client{Timeout: 30 * time.Second}, + } +} + +// getTenantAccessToken retrieves (or returns cached) tenant access token. +// Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin. +func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) { + c.tokenMu.Lock() + defer c.tokenMu.Unlock() + + if c.tokenCache != "" && time.Now().Before(c.tokenExpAt) { + return c.tokenCache, nil + } + + payload, _ := json.Marshal(map[string]string{ + "app_id": c.appID, + "app_secret": c.appSecret, + }) + + url := c.baseURL + "/open-apis/auth/v3/tenant_access_token/internal" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("create token request: %w", err) + } + req.Header.Set("Content-Type", "application/json; charset=utf-8") + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("request token: %w", err) + } + defer resp.Body.Close() + + var result tokenResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("decode token response: %w", err) + } + if result.Code != 0 { + return "", fmt.Errorf("feishu auth error: code=%d msg=%s", result.Code, result.Msg) + } + + c.tokenCache = result.TenantAccessToken + ttl := time.Duration(result.Expire) * time.Second + if ttl > 5*time.Minute { + ttl -= 5 * time.Minute + } + c.tokenExpAt = time.Now().Add(ttl) + + logger.Infof(ctx, "[Feishu] got tenant_access_token: %s...%s expire=%ds", + result.TenantAccessToken[:8], result.TenantAccessToken[len(result.TenantAccessToken)-4:], result.Expire) + + return c.tokenCache, nil +} + +// doRequest executes an authenticated API request and decodes the JSON response. +func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error { + token, err := c.getTenantAccessToken(ctx) + if err != nil { + return err + } + + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + url := c.baseURL + path + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + return fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Authorization", "Bearer "+token) + + logger.Infof(ctx, "[Feishu] %s %s", method, path) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("execute request: %w", err) + } + defer resp.Body.Close() + + // Read body once for logging + decoding + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read response body: %w", err) + } + + logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s", + method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000)) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody)) + } + + if result != nil { + if err := json.Unmarshal(respBody, result); err != nil { + return fmt.Errorf("decode response: %w", err) + } + } + + return nil +} + +// truncate truncates a string to maxLen and appends "..." if truncated. +func truncate(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} + +// ListWikiSpaces returns all wiki spaces accessible to the app. +func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) { + var allSpaces []wikiSpace + pageToken := "" + + for { + path := "/open-apis/wiki/v2/spaces?page_size=50" + if pageToken != "" { + path += "&page_token=" + pageToken + } + + var resp wikiSpaceListResponse + if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return nil, fmt.Errorf("list wiki spaces: %w", err) + } + if resp.Code != 0 { + logger.Errorf(ctx, "[Feishu] ListWikiSpaces error: code=%d msg=%s", resp.Code, resp.Msg) + return nil, fmt.Errorf("list wiki spaces error: code=%d msg=%s", resp.Code, resp.Msg) + } + + logger.Infof(ctx, "[Feishu] ListWikiSpaces: got %d spaces, has_more=%v", len(resp.Data.Items), resp.Data.HasMore) + for i, s := range resp.Data.Items { + logger.Infof(ctx, "[Feishu] space[%d]: id=%s name=%q visibility=%s", i, s.SpaceID, s.Name, s.Visibility) + } + + allSpaces = append(allSpaces, resp.Data.Items...) + + if !resp.Data.HasMore || resp.Data.PageToken == "" { + break + } + pageToken = resp.Data.PageToken + } + + logger.Infof(ctx, "[Feishu] ListWikiSpaces: total %d spaces", len(allSpaces)) + return allSpaces, nil +} + +// ListWikiNodes returns all nodes (documents) under a wiki space. +// If parentNodeToken is empty, returns top-level nodes. +func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]wikiNode, error) { + var allNodes []wikiNode + pageToken := "" + + for { + path := fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes?page_size=50", spaceID) + if parentNodeToken != "" { + path += "&parent_node_token=" + parentNodeToken + } + if pageToken != "" { + path += "&page_token=" + pageToken + } + + var resp wikiNodeListResponse + if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return nil, fmt.Errorf("list wiki nodes: %w", err) + } + if resp.Code != 0 { + return nil, fmt.Errorf("list wiki nodes error: code=%d msg=%s", resp.Code, resp.Msg) + } + + allNodes = append(allNodes, resp.Data.Items...) + + if !resp.Data.HasMore || resp.Data.PageToken == "" { + break + } + pageToken = resp.Data.PageToken + } + + return allNodes, nil +} + +// ListAllWikiNodesRecursive recursively lists all nodes under a wiki space. +// It walks the tree depth-first to discover all nested documents. +func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]wikiNode, error) { + // Start with top-level nodes + topNodes, err := c.ListWikiNodes(ctx, spaceID, "") + if err != nil { + return nil, err + } + + var allNodes []wikiNode + var walk func(nodes []wikiNode) error + + walk = func(nodes []wikiNode) error { + for _, node := range nodes { + allNodes = append(allNodes, node) + + // Recurse into child nodes if this node has children + if node.HasChild { + children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken) + if err != nil { + return fmt.Errorf("list children of %s: %w", node.NodeToken, err) + } + if err := walk(children); err != nil { + return err + } + } + } + return nil + } + + if err := walk(topNodes); err != nil { + return nil, err + } + + return allNodes, nil +} + +// GetDocumentRawContent retrieves the raw text content of a Feishu docx document. +// This returns plain text (not rich text / block structure). +// Deprecated: prefer ExportAndDownload which preserves formatting. +func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) (string, error) { + path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID) + + var resp docRawContentResponse + if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return "", fmt.Errorf("get document raw content: %w", err) + } + if resp.Code != 0 { + return "", fmt.Errorf("get document raw content error: code=%d msg=%s", resp.Code, resp.Msg) + } + + return resp.Data.Content, nil +} + +// Ping verifies the credentials by attempting to get a tenant access token. +func (c *Client) Ping(ctx context.Context) error { + _, err := c.getTenantAccessToken(ctx) + return err +} + +// ────────────────────────────────────────────────────────────────────── +// Export task API: export docx/sheet/bitable to downloadable files +// +// Flow: +// 1. POST /drive/v1/export_tasks → create export task, get ticket +// 2. GET /drive/v1/export_tasks/:ticket → poll until status=0 (success) +// 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes +// ────────────────────────────────────────────────────────────────────── + +// CreateExportTask creates an async export task for a Feishu document. +// - token: the obj_token of the document (e.g. docx token, sheet token) +// - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable") +// - fileExtension: desired output format ("docx", "xlsx", "pdf") +func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) { + body := map[string]string{ + "file_extension": fileExtension, + "token": token, + "type": objType, + } + + var resp exportTaskCreateResponse + if err := c.doRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil { + return "", fmt.Errorf("create export task: %w", err) + } + if resp.Code != 0 { + return "", fmt.Errorf("create export task error: code=%d msg=%s", resp.Code, resp.Msg) + } + + return resp.Data.Ticket, nil +} + +// GetExportTaskStatus polls the status of an export task. +// Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds. +// The token parameter is the obj_token of the document being exported (required by the API). +func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) { + path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token) + + var resp exportTaskStatusResponse + if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return "", "", fmt.Errorf("get export task status: %w", err) + } + if resp.Code != 0 { + return "", "", fmt.Errorf("get export task status error: code=%d msg=%s", resp.Code, resp.Msg) + } + + r := resp.Data.Result + switch r.JobStatus { + case 0: // success + return r.FileToken, r.FileName, nil + case 1, 2: // initializing, processing + return "", "", nil // not ready yet + default: + return "", "", fmt.Errorf("export task failed: status=%d msg=%s", r.JobStatus, r.JobErrorMsg) + } +} + +// DownloadExportFile downloads the exported file by its file_token. +// The file_token is returned by GetExportTaskStatus when the export job completes. +// The file must be downloaded within 10 minutes of export completion. +func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]byte, error) { + path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken) + return c.downloadRawBytes(ctx, path) +} + +// ExportAndDownload is a high-level helper that creates an export task, polls until +// completion, and downloads the resulting file. Returns (fileBytes, fileName, error). +// +// Timeout: 60 seconds. Poll interval: 2 seconds. +func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) { + // Determine export format + fileExt, ok := objTypeToExportFileExtension[objType] + if !ok { + return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType) + } + + exportType, ok := objTypeToExportType[objType] + if !ok { + return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType) + } + + // Step 1: create export task + ticket, err := c.CreateExportTask(ctx, objToken, exportType, fileExt) + if err != nil { + return nil, "", err + } + + // Step 2: poll until ready (max 60s, every 2s) + deadline := time.Now().Add(60 * time.Second) + var fileToken, fileName string + + for time.Now().Before(deadline) { + fileToken, fileName, err = c.GetExportTaskStatus(ctx, ticket, objToken) + if err != nil { + return nil, "", err + } + if fileToken != "" { + break // export ready + } + select { + case <-ctx.Done(): + return nil, "", ctx.Err() + case <-time.After(2 * time.Second): + } + } + + if fileToken == "" { + return nil, "", fmt.Errorf("export task timed out after 60s (ticket=%s)", ticket) + } + + // Step 3: download file using file_token (NOT ticket) + data, err := c.DownloadExportFile(ctx, fileToken) + if err != nil { + return nil, "", err + } + + // Build a sensible file name + if fileName == "" { + fileName = "export" + exportFileExtToSuffix[fileExt] + } + + return data, fileName, nil +} + +// ────────────────────────────────────────────────────────────────────── +// Drive file download: for "file" type wiki nodes (uploaded PDF/Word/etc.) +// ────────────────────────────────────────────────────────────────────── + +// DownloadDriveFile downloads a file from Feishu Drive by its file token. +// Used for wiki nodes with obj_type="file" (user-uploaded PDF, Word, images, etc.). +func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byte, error) { + path := fmt.Sprintf("/open-apis/drive/v1/files/%s/download", fileToken) + return c.downloadRawBytes(ctx, path) +} + +// downloadRawBytes performs an authenticated GET and returns the raw response body. +func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) { + token, err := c.getTenantAccessToken(ctx) + if err != nil { + return nil, err + } + + url := c.baseURL + path + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("create download request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + + logger.Infof(ctx, "[Feishu] download GET %s", path) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("download request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500)) + return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body)) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read download body: %w", err) + } + + logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data)) + return data, nil +} diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go new file mode 100644 index 00000000..2f45417e --- /dev/null +++ b/internal/datasource/connector/feishu/connector.go @@ -0,0 +1,376 @@ +package feishu + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Connector implements the datasource.Connector interface for Feishu. +type Connector struct{} + +// NewConnector creates a new Feishu connector. +func NewConnector() *Connector { + return &Connector{} +} + +// Type returns the connector type identifier. +func (c *Connector) Type() string { + return types.ConnectorTypeFeishu +} + +// Validate verifies that the Feishu configuration is valid by testing connectivity. +func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error { + feishuConfig, err := parseFeishuConfig(config) + if err != nil { + return err + } + + client := NewClient(feishuConfig) + if err := client.Ping(ctx); err != nil { + return fmt.Errorf("feishu connection failed: %w", err) + } + + return nil +} + +// ListResources lists all accessible Feishu Wiki spaces as selectable resources. +func (c *Connector) ListResources(ctx context.Context, config *types.DataSourceConfig) ([]types.Resource, error) { + feishuConfig, err := parseFeishuConfig(config) + if err != nil { + return nil, err + } + + client := NewClient(feishuConfig) + spaces, err := client.ListWikiSpaces(ctx) + if err != nil { + return nil, fmt.Errorf("list feishu wiki spaces: %w", err) + } + + resources := make([]types.Resource, 0, len(spaces)) + for _, space := range spaces { + resources = append(resources, types.Resource{ + ExternalID: space.SpaceID, + Name: space.Name, + Type: "wiki_space", + Description: space.Description, + URL: fmt.Sprintf("https://feishu.cn/wiki/%s", space.SpaceID), + Metadata: map[string]interface{}{ + "visibility": space.Visibility, + }, + }) + } + + return resources, nil +} + +// FetchAll performs a full sync of all documents from the specified wiki spaces. +func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) { + feishuConfig, err := parseFeishuConfig(config) + if err != nil { + return nil, err + } + + client := NewClient(feishuConfig) + + var allItems []types.FetchedItem + + for _, spaceID := range resourceIDs { + // List all nodes in this wiki space recursively + nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID) + if err != nil { + return nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + } + + // Fetch content for each document node + for _, node := range nodes { + item, err := c.fetchNodeContent(ctx, client, node, spaceID) + if err != nil { + // Log error but continue with other nodes + allItems = append(allItems, types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + SourceResourceID: spaceID, + Metadata: map[string]string{ + "error": err.Error(), + }, + }) + continue + } + if item != nil { + allItems = append(allItems, *item) + } + } + } + + return allItems, nil +} + +// FetchIncremental performs an incremental sync by comparing node edit times +// against the previously recorded state. +func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) { + feishuConfig, err := parseFeishuConfig(config) + if err != nil { + return nil, nil, err + } + + client := NewClient(feishuConfig) + + // Parse the previous cursor state + var prevCursor feishuCursor + if cursor != nil && cursor.ConnectorCursor != nil { + cursorBytes, _ := json.Marshal(cursor.ConnectorCursor) + _ = json.Unmarshal(cursorBytes, &prevCursor) + } + + // Build new cursor to track current state + newCursor := feishuCursor{ + LastSyncTime: time.Now(), + SpaceNodeTimes: make(map[string]map[string]string), + } + + var changedItems []types.FetchedItem + + // Get resource IDs from config + resourceIDs := config.ResourceIDs + if len(resourceIDs) == 0 { + return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs) configured") + } + + for _, spaceID := range resourceIDs { + // List all nodes in this space + nodes, err := client.ListAllWikiNodesRecursive(ctx, spaceID) + if err != nil { + return nil, nil, fmt.Errorf("list nodes in space %s: %w", spaceID, err) + } + + newCursor.SpaceNodeTimes[spaceID] = make(map[string]string) + + // Build a set of current node tokens for deletion detection + currentNodes := make(map[string]bool) + + for _, node := range nodes { + currentNodes[node.NodeToken] = true + // Use ObjEditTime (document content edit time) for change detection, + // NOT NodeEditTime which only tracks node attribute changes (title, position). + editTimeStr := node.ObjEditTime + if editTimeStr == "" { + editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time + } + newCursor.SpaceNodeTimes[spaceID][node.NodeToken] = editTimeStr + + // Check if node has changed since last sync + if prevCursor.SpaceNodeTimes != nil { + if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok { + if prevEditTime, exists := prevTimes[node.NodeToken]; exists { + if prevEditTime == editTimeStr { + // Node unchanged, skip + continue + } + } + } + } + + // Node is new or changed — fetch its content + item, err := c.fetchNodeContent(ctx, client, node, spaceID) + if err != nil { + // Record failed items + changedItems = append(changedItems, types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + SourceResourceID: spaceID, + Metadata: map[string]string{ + "error": err.Error(), + }, + }) + continue + } + if item != nil { + changedItems = append(changedItems, *item) + } + } + + // Detect deleted nodes + if prevCursor.SpaceNodeTimes != nil { + if prevTimes, ok := prevCursor.SpaceNodeTimes[spaceID]; ok { + for nodeToken := range prevTimes { + if !currentNodes[nodeToken] { + // Node was deleted + changedItems = append(changedItems, types.FetchedItem{ + ExternalID: nodeToken, + IsDeleted: true, + SourceResourceID: spaceID, + }) + } + } + } + } + } + + // Build next sync cursor + nextCursorMap := make(map[string]interface{}) + cursorBytes, _ := json.Marshal(newCursor) + _ = json.Unmarshal(cursorBytes, &nextCursorMap) + + nextSyncCursor := &types.SyncCursor{ + LastSyncTime: time.Now(), + ConnectorCursor: nextCursorMap, + } + + return changedItems, nextSyncCursor, nil +} + +// fetchNodeContent fetches the content of a single wiki node and converts it to FetchedItem. +// Dispatches to different retrieval strategies based on obj_type: +// - docx/doc → export API → .docx file +// - sheet → export API → .xlsx file +// - bitable → export API → .xlsx file +// - file → drive download → original file (PDF/Word/image/etc.) +// - mindnote → skip (no API) +// - slides → skip (no API) +func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string) (*types.FetchedItem, error) { + if !isSupportedDocType(node.ObjType) { + return nil, nil + } + + editTime := parseFeishuTimestamp(node.NodeEditTime) + baseMeta := map[string]string{ + "obj_token": node.ObjToken, + "obj_type": node.ObjType, + "node_token": node.NodeToken, + "space_id": spaceID, + "creator": node.Creator, + "owner": node.Owner, + "channel": types.ChannelFeishu, + } + + switch node.ObjType { + case "docx", "doc", "sheet", "bitable": + // Export as a file via the async export API + data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType) + if err != nil { + return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err) + } + + // Ensure a reasonable file name with correct extension + ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]] + if fileName == "" { + fileName = sanitizeFileName(node.Title) + ext + } else if !strings.HasSuffix(strings.ToLower(fileName), ext) { + // Feishu often returns the doc title without extension — append it + fileName = sanitizeFileName(fileName) + ext + } + + return &types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken), + UpdatedAt: editTime, + SourceResourceID: spaceID, + Metadata: baseMeta, + }, nil + + case "file": + // Download the original uploaded file from Drive + data, err := client.DownloadDriveFile(ctx, node.ObjToken) + if err != nil { + return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err) + } + + // Use the node title as file name; it usually preserves the original extension + fileName := node.Title + if fileName == "" { + fileName = node.ObjToken + } + + return &types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: fmt.Sprintf("https://feishu.cn/wiki/%s", node.NodeToken), + UpdatedAt: editTime, + SourceResourceID: spaceID, + Metadata: baseMeta, + }, nil + + default: + return nil, nil + } +} + +// --- Helper functions --- + +// parseFeishuConfig extracts and validates Feishu-specific configuration. +func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) { + if config == nil { + return nil, fmt.Errorf("config is nil") + } + + credBytes, err := json.Marshal(config.Credentials) + if err != nil { + return nil, fmt.Errorf("marshal credentials: %w", err) + } + + var feishuConfig Config + if err := json.Unmarshal(credBytes, &feishuConfig); err != nil { + return nil, fmt.Errorf("parse feishu credentials: %w", err) + } + + if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" { + return nil, fmt.Errorf("feishu app_id and app_secret are required") + } + + return &feishuConfig, nil +} + +// isSupportedDocType checks if a Feishu document type can be synced. +// mindnote and slides have no content read API and are skipped. +func isSupportedDocType(objType string) bool { + switch objType { + case "docx", "doc", "sheet", "bitable", "file": + return true + default: + // mindnote, slides — no content retrieval API available + return false + } +} + +// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time. +func parseFeishuTimestamp(ts string) time.Time { + if ts == "" { + return time.Time{} + } + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return time.Time{} + } + return time.Unix(sec, 0) +} + +// sanitizeFileName removes characters that are invalid in filenames. +func sanitizeFileName(name string) string { + if name == "" { + return "untitled" + } + // Replace common invalid characters with underscore + replacer := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", + ) + result := replacer.Replace(name) + // Limit length + if len(result) > 200 { + result = result[:200] + } + return result +} diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go new file mode 100644 index 00000000..b74092cc --- /dev/null +++ b/internal/datasource/connector/feishu/connector_test.go @@ -0,0 +1,817 @@ +package feishu + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +// ────────────────────────────────────────────────────────────────────── +// Fake Feishu API server +// ────────────────────────────────────────────────────────────────────── + +// fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs. +// It returns the server and a Config pointing at it. +func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { + mux := http.NewServeMux() + + // --- auth --- + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, tokenResponse{ + apiResponse: apiResponse{Code: 0}, + TenantAccessToken: "fake-token", + Expire: 7200, + }) + }) + + // --- wiki spaces --- + mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, wikiSpaceListResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct { + Items []wikiSpace `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + }{ + Items: []wikiSpace{ + {SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"}, + }, + }, + }) + }) + + // --- wiki nodes (top-level only for simplicity) --- + mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, wikiNodeListResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct { + Items []wikiNode `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + }{ + Items: nodes, + }, + }) + }) + + // --- export task: create --- + mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + writeJSON(w, exportTaskCreateResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct{ Ticket string `json:"ticket"` }{Ticket: "ticket-123"}, + }) + return + } + // GET /open-apis/drive/v1/export_tasks/ticket-123 + writeJSON(w, exportTaskStatusResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct { + Result struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` + } `json:"result"` + }{ + Result: struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` + }{ + FileToken: "ft-abc", + FileSize: 100, + JobStatus: 0, // success + FileName: "exported.docx", + }, + }, + }) + }) + + // --- export task: status polling (pattern match with ticket) --- + mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, exportTaskStatusResponse{ + apiResponse: apiResponse{Code: 0}, + Data: struct { + Result struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` + } `json:"result"` + }{ + Result: struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` + }{ + FileToken: "ft-abc", + FileSize: 100, + JobStatus: 0, + FileName: "exported.docx", + }, + }, + }) + }) + + // --- export file download --- + mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ticket-123/download", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write([]byte("fake-docx-content")) + }) + + // --- drive file download (for "file" type nodes) --- + mux.HandleFunc("/open-apis/drive/v1/files/", func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/download") { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write([]byte("fake-pdf-binary")) + return + } + http.NotFound(w, r) + }) + + ts := httptest.NewServer(mux) + cfg := &Config{ + AppID: "test-app-id", + AppSecret: "test-app-secret", + BaseURL: ts.URL, + } + return ts, cfg +} + +func writeJSON(w http.ResponseWriter, v interface{}) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func makeConfig(cfg *Config, resourceIDs []string) *types.DataSourceConfig { + creds := map[string]interface{}{ + "app_id": cfg.AppID, + "app_secret": cfg.AppSecret, + "base_url": cfg.BaseURL, + } + return &types.DataSourceConfig{ + Type: types.ConnectorTypeFeishu, + Credentials: creds, + ResourceIDs: resourceIDs, + } +} + +// ────────────────────────────────────────────────────────────────────── +// Helper function tests +// ────────────────────────────────────────────────────────────────────── + +func TestIsSupportedDocType(t *testing.T) { + tests := []struct { + objType string + expected bool + }{ + {"docx", true}, + {"doc", true}, + {"sheet", true}, + {"bitable", true}, + {"file", true}, + {"mindnote", false}, + {"slides", false}, + {"unknown", false}, + {"", false}, + } + + for _, tt := range tests { + t.Run(tt.objType, func(t *testing.T) { + got := isSupportedDocType(tt.objType) + if got != tt.expected { + t.Errorf("isSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected) + } + }) + } +} + +func TestSanitizeFileName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"hello", "hello"}, + {"", "untitled"}, + {"a/b\\c:d*e", "a_b_c_d_e"}, + {"normal file.docx", "normal file.docx"}, + {strings.Repeat("a", 300), strings.Repeat("a", 200)}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := sanitizeFileName(tt.input) + if got != tt.expected { + t.Errorf("sanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } +} + +func TestParseFeishuTimestamp(t *testing.T) { + ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC + if ts.IsZero() { + t.Fatal("expected non-zero time") + } + if ts.Unix() != 1711468800 { + t.Errorf("unexpected unix = %d", ts.Unix()) + } + + if !parseFeishuTimestamp("").IsZero() { + t.Error("expected zero time for empty string") + } + if !parseFeishuTimestamp("invalid").IsZero() { + t.Error("expected zero time for invalid string") + } +} + +func TestParseFeishuConfig(t *testing.T) { + t.Run("valid", func(t *testing.T) { + cfg, err := parseFeishuConfig(&types.DataSourceConfig{ + Credentials: map[string]interface{}{ + "app_id": "id1", + "app_secret": "sec1", + "base_url": "https://custom.example.com", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" { + t.Errorf("unexpected config: %+v", cfg) + } + }) + + t.Run("nil config", func(t *testing.T) { + _, err := parseFeishuConfig(nil) + if err == nil { + t.Fatal("expected error for nil config") + } + }) + + t.Run("missing credentials", func(t *testing.T) { + _, err := parseFeishuConfig(&types.DataSourceConfig{ + Credentials: map[string]interface{}{ + "app_id": "id1", + // missing app_secret + }, + }) + if err == nil { + t.Fatal("expected error for missing app_secret") + } + }) +} + +// ────────────────────────────────────────────────────────────────────── +// Connector interface tests +// ────────────────────────────────────────────────────────────────────── + +func TestConnectorType(t *testing.T) { + c := NewConnector() + if c.Type() != types.ConnectorTypeFeishu { + t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu) + } +} + +func TestConnectorValidate(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + c := NewConnector() + err := c.Validate(context.Background(), makeConfig(cfg, nil)) + if err != nil { + t.Fatalf("Validate() error: %v", err) + } +} + +func TestConnectorValidate_BadCredentials(t *testing.T) { + c := NewConnector() + err := c.Validate(context.Background(), &types.DataSourceConfig{ + Credentials: map[string]interface{}{ + "app_id": "bad", + "app_secret": "bad", + "base_url": "http://127.0.0.1:1", // will fail to connect + }, + }) + if err == nil { + t.Fatal("expected error for bad credentials") + } +} + +func TestConnectorListResources(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + c := NewConnector() + resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil)) + if err != nil { + t.Fatalf("ListResources() error: %v", err) + } + + if len(resources) != 1 { + t.Fatalf("expected 1 resource, got %d", len(resources)) + } + if resources[0].ExternalID != "space1" { + t.Errorf("ExternalID = %q, want %q", resources[0].ExternalID, "space1") + } + if resources[0].Name != "Test Space" { + t.Errorf("Name = %q, want %q", resources[0].Name, "Test Space") + } + if resources[0].Type != "wiki_space" { + t.Errorf("Type = %q, want %q", resources[0].Type, "wiki_space") + } +} + +// ────────────────────────────────────────────────────────────────────── +// FetchAll: test all supported doc types +// ────────────────────────────────────────────────────────────────────── + +func TestFetchAll_DocxNode(t *testing.T) { + nodes := []wikiNode{{ + NodeToken: "nt1", + ObjToken: "obj-docx-1", + ObjType: "docx", + Title: "My Document", + NodeEditTime: "1711468800", + }} + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + + item := items[0] + if item.ExternalID != "nt1" { + t.Errorf("ExternalID = %q, want %q", item.ExternalID, "nt1") + } + if item.Title != "My Document" { + t.Errorf("Title = %q", item.Title) + } + if string(item.Content) != "fake-docx-content" { + t.Errorf("Content = %q, want %q", string(item.Content), "fake-docx-content") + } + if item.FileName != "exported.docx" { + t.Errorf("FileName = %q, want %q", item.FileName, "exported.docx") + } + if item.Metadata["obj_type"] != "docx" { + t.Errorf("Metadata[obj_type] = %q", item.Metadata["obj_type"]) + } + if item.Metadata["channel"] != types.ChannelFeishu { + t.Errorf("Metadata[channel] = %q", item.Metadata["channel"]) + } +} + +func TestFetchAll_SheetNode(t *testing.T) { + nodes := []wikiNode{{ + NodeToken: "nt-sheet", + ObjToken: "obj-sheet-1", + ObjType: "sheet", + Title: "Sales Report", + NodeEditTime: "1711468800", + }} + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].Metadata["obj_type"] != "sheet" { + t.Errorf("obj_type = %q, want sheet", items[0].Metadata["obj_type"]) + } +} + +func TestFetchAll_BitableNode(t *testing.T) { + nodes := []wikiNode{{ + NodeToken: "nt-bitable", + ObjToken: "obj-bitable-1", + ObjType: "bitable", + Title: "Project Tracker", + NodeEditTime: "1711468800", + }} + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].Metadata["obj_type"] != "bitable" { + t.Errorf("obj_type = %q, want bitable", items[0].Metadata["obj_type"]) + } +} + +func TestFetchAll_FileNode(t *testing.T) { + nodes := []wikiNode{{ + NodeToken: "nt-file", + ObjToken: "obj-file-1", + ObjType: "file", + Title: "manual.pdf", + NodeEditTime: "1711468800", + }} + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + + item := items[0] + if string(item.Content) != "fake-pdf-binary" { + t.Errorf("Content = %q, want %q", string(item.Content), "fake-pdf-binary") + } + if item.FileName != "manual.pdf" { + t.Errorf("FileName = %q, want %q", item.FileName, "manual.pdf") + } + if item.Metadata["obj_type"] != "file" { + t.Errorf("obj_type = %q, want file", item.Metadata["obj_type"]) + } +} + +func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"}, + {NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"}, + } + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + + // Both should be skipped (nil returned by fetchNodeContent) + if len(items) != 0 { + t.Errorf("expected 0 items (mindnote+slides skipped), got %d", len(items)) + } +} + +func TestFetchAll_MixedTypes(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"}, + {NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"}, + {NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"}, + {NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"}, + {NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"}, + {NodeToken: "nt6", ObjToken: "obj6", ObjType: "bitable", Title: "Table", NodeEditTime: "1711468800"}, + } + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) + if err != nil { + t.Fatalf("FetchAll() error: %v", err) + } + + // docx + sheet + file + bitable = 4 items; mindnote + slides = skipped + if len(items) != 4 { + t.Errorf("expected 4 items, got %d", len(items)) + for i, it := range items { + t.Logf(" item[%d]: %s (obj_type=%s)", i, it.Title, it.Metadata["obj_type"]) + } + } +} + +// ────────────────────────────────────────────────────────────────────── +// FetchIncremental tests +// ────────────────────────────────────────────────────────────────────── + +func TestFetchIncremental_FirstSync(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, + {NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"}, + } + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + dsConfig := makeConfig(cfg, []string{"space1"}) + + // First sync with no cursor → all items should be fetched + items, cursor, err := c.FetchIncremental(context.Background(), dsConfig, nil) + if err != nil { + t.Fatalf("FetchIncremental() error: %v", err) + } + + if len(items) != 2 { + t.Fatalf("expected 2 items on first sync, got %d", len(items)) + } + if cursor == nil { + t.Fatal("expected non-nil cursor") + } + if cursor.LastSyncTime.IsZero() { + t.Error("cursor.LastSyncTime should not be zero") + } +} + +func TestFetchIncremental_NoChanges(t *testing.T) { + nodes := []wikiNode{ + {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, + } + ts, cfg := fakeFeishu(nodes) + defer ts.Close() + + c := NewConnector() + dsConfig := makeConfig(cfg, []string{"space1"}) + + // First sync + _, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil) + if err != nil { + t.Fatalf("first sync error: %v", err) + } + + // Second sync with same edit times → should return 0 changed items + items, _, err := c.FetchIncremental(context.Background(), dsConfig, cursor1) + if err != nil { + t.Fatalf("second sync error: %v", err) + } + + if len(items) != 0 { + t.Errorf("expected 0 items (no changes), got %d", len(items)) + } +} + +func TestFetchIncremental_DetectsDeleted(t *testing.T) { + // First sync: 2 nodes + allNodes := []wikiNode{ + {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, + {NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"}, + } + ts, cfg := fakeFeishu(allNodes) + + c := NewConnector() + dsConfig := makeConfig(cfg, []string{"space1"}) + + _, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil) + if err != nil { + t.Fatalf("first sync error: %v", err) + } + ts.Close() + + // Second sync: only 1 node remains (nt2 was deleted) + ts2, cfg2 := fakeFeishu([]wikiNode{ + {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, + }) + defer ts2.Close() + dsConfig2 := makeConfig(cfg2, []string{"space1"}) + + items, _, err := c.FetchIncremental(context.Background(), dsConfig2, cursor1) + if err != nil { + t.Fatalf("second sync error: %v", err) + } + + // Should have 1 deleted item for nt2 + var deletedCount int + for _, item := range items { + if item.IsDeleted { + deletedCount++ + if item.ExternalID != "nt2" { + t.Errorf("expected deleted ExternalID=nt2, got %q", item.ExternalID) + } + } + } + if deletedCount != 1 { + t.Errorf("expected 1 deleted item, got %d", deletedCount) + } +} + +func TestFetchIncremental_NoResourceIDs(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + c := NewConnector() + dsConfig := makeConfig(cfg, nil) // no resource IDs + dsConfig.ResourceIDs = nil + + _, _, err := c.FetchIncremental(context.Background(), dsConfig, nil) + if err == nil { + t.Fatal("expected error for empty resource IDs") + } + if !strings.Contains(err.Error(), "no resource IDs") { + t.Errorf("unexpected error: %v", err) + } +} + +// ────────────────────────────────────────────────────────────────────── +// Client tests +// ────────────────────────────────────────────────────────────────────── + +func TestClientPing(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + client := NewClient(cfg) + if err := client.Ping(context.Background()); err != nil { + t.Fatalf("Ping() error: %v", err) + } +} + +func TestClientTokenCaching(t *testing.T) { + callCount := 0 + mux := http.NewServeMux() + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { + callCount++ + writeJSON(w, tokenResponse{ + apiResponse: apiResponse{Code: 0}, + TenantAccessToken: fmt.Sprintf("token-%d", callCount), + Expire: 7200, + }) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + client := NewClient(&Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}) + + // First call: fetches token + t1, _ := client.getTenantAccessToken(context.Background()) + // Second call: should use cache + t2, _ := client.getTenantAccessToken(context.Background()) + + if t1 != t2 { + t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2) + } + if callCount != 1 { + t.Errorf("expected 1 auth call, got %d", callCount) + } +} + +func TestClientExportAndDownload(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + client := NewClient(cfg) + data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx") + if err != nil { + t.Fatalf("ExportAndDownload() error: %v", err) + } + + if string(data) != "fake-docx-content" { + t.Errorf("data = %q, want %q", string(data), "fake-docx-content") + } + if fileName != "exported.docx" { + t.Errorf("fileName = %q, want %q", fileName, "exported.docx") + } +} + +func TestClientExportAndDownload_UnsupportedType(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + client := NewClient(cfg) + _, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote") + if err == nil { + t.Fatal("expected error for unsupported type") + } + if !strings.Contains(err.Error(), "unsupported") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestClientDownloadDriveFile(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + client := NewClient(cfg) + data, err := client.DownloadDriveFile(context.Background(), "file-token-abc") + if err != nil { + t.Fatalf("DownloadDriveFile() error: %v", err) + } + + if string(data) != "fake-pdf-binary" { + t.Errorf("data = %q, want %q", string(data), "fake-pdf-binary") + } +} + +func TestClientListWikiSpaces(t *testing.T) { + ts, cfg := fakeFeishu(nil) + defer ts.Close() + + client := NewClient(cfg) + spaces, err := client.ListWikiSpaces(context.Background()) + if err != nil { + t.Fatalf("ListWikiSpaces() error: %v", err) + } + if len(spaces) != 1 { + t.Fatalf("expected 1 space, got %d", len(spaces)) + } + if spaces[0].SpaceID != "space1" { + t.Errorf("SpaceID = %q", spaces[0].SpaceID) + } +} + +// ────────────────────────────────────────────────────────────────────── +// Type mapping tests +// ────────────────────────────────────────────────────────────────────── + +func TestObjTypeToExportMappings(t *testing.T) { + // Verify all exportable types have valid mappings + exportable := []string{"docx", "doc", "sheet", "bitable"} + for _, ot := range exportable { + if _, ok := objTypeToExportFileExtension[ot]; !ok { + t.Errorf("objTypeToExportFileExtension missing %q", ot) + } + if _, ok := objTypeToExportType[ot]; !ok { + t.Errorf("objTypeToExportType missing %q", ot) + } + } + + // Verify non-exportable types do NOT have mappings + nonExportable := []string{"file", "mindnote", "slides"} + for _, ot := range nonExportable { + if _, ok := objTypeToExportFileExtension[ot]; ok { + t.Errorf("objTypeToExportFileExtension should NOT contain %q", ot) + } + } +} + +func TestExportFileExtToSuffix(t *testing.T) { + if exportFileExtToSuffix[ExportTypeDocx] != ".docx" { + t.Errorf("docx suffix = %q", exportFileExtToSuffix[ExportTypeDocx]) + } + if exportFileExtToSuffix[ExportTypeXlsx] != ".xlsx" { + t.Errorf("xlsx suffix = %q", exportFileExtToSuffix[ExportTypeXlsx]) + } + if exportFileExtToSuffix[ExportTypePDF] != ".pdf" { + t.Errorf("pdf suffix = %q", exportFileExtToSuffix[ExportTypePDF]) + } +} + +// ────────────────────────────────────────────────────────────────────── +// Feishu cursor serialization +// ────────────────────────────────────────────────────────────────────── + +func TestFeishuCursorRoundTrip(t *testing.T) { + original := feishuCursor{ + LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC), + SpaceNodeTimes: map[string]map[string]string{ + "space1": { + "nt1": "100", + "nt2": "200", + }, + }, + } + + // Serialize to map[string]interface{} (as stored in SyncCursor.ConnectorCursor) + data, err := json.Marshal(original) + if err != nil { + t.Fatalf("marshal error: %v", err) + } + + var cursorMap map[string]interface{} + if err := json.Unmarshal(data, &cursorMap); err != nil { + t.Fatalf("unmarshal to map error: %v", err) + } + + // Deserialize back + data2, _ := json.Marshal(cursorMap) + var restored feishuCursor + if err := json.Unmarshal(data2, &restored); err != nil { + t.Fatalf("restore error: %v", err) + } + + if restored.SpaceNodeTimes["space1"]["nt1"] != "100" { + t.Errorf("restored nt1 = %q, want 100", restored.SpaceNodeTimes["space1"]["nt1"]) + } + if restored.SpaceNodeTimes["space1"]["nt2"] != "200" { + t.Errorf("restored nt2 = %q, want 200", restored.SpaceNodeTimes["space1"]["nt2"]) + } +} diff --git a/internal/datasource/connector/feishu/types.go b/internal/datasource/connector/feishu/types.go new file mode 100644 index 00000000..148b83ff --- /dev/null +++ b/internal/datasource/connector/feishu/types.go @@ -0,0 +1,206 @@ +// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora. +// +// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases. +// +// Feishu API docs: +// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list +// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list +// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide +// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download +// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal +package feishu + +import "time" + +// Config holds Feishu-specific configuration for the data source connector. +// Uses the self-built app (企业自建应用) authentication model. +type Config struct { + // App ID from Feishu developer console + AppID string `json:"app_id"` + + // App Secret from Feishu developer console + AppSecret string `json:"app_secret"` + + // Base URL for Feishu API (default: https://open.feishu.cn) + // Use https://open.larksuite.com for Lark (international) deployments + BaseURL string `json:"base_url,omitempty"` +} + +// DefaultBaseURL is the default Feishu Open Platform API base URL. +const DefaultBaseURL = "https://open.feishu.cn" + +// LarkBaseURL is the Lark (international) API base URL. +const LarkBaseURL = "https://open.larksuite.com" + +// GetBaseURL returns the effective base URL, defaulting to Feishu if not set. +func (c *Config) GetBaseURL() string { + if c.BaseURL != "" { + return c.BaseURL + } + return DefaultBaseURL +} + +// --- Export format constants --- +// Used by the export task API: POST /drive/v1/export_tasks + +const ( + // ExportTypeDocx exports Feishu documents to .docx format. + ExportTypeDocx = "docx" + // ExportTypeXlsx exports spreadsheets / bitable to .xlsx format. + ExportTypeXlsx = "xlsx" + // ExportTypePDF exports documents to .pdf format (fallback). + ExportTypePDF = "pdf" +) + +// objTypeToExportFileExtension maps Feishu obj_type to the best export file_extension. +var objTypeToExportFileExtension = map[string]string{ + "docx": ExportTypeDocx, + "doc": ExportTypeDocx, + "sheet": ExportTypeXlsx, + "bitable": ExportTypeXlsx, +} + +// objTypeToExportType maps Feishu obj_type to the export API "type" parameter. +// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create +var objTypeToExportType = map[string]string{ + "docx": "docx", + "doc": "doc", + "sheet": "sheet", + "bitable": "bitable", +} + +// exportFileExtToSuffix maps export file_extension to the file suffix for FileName. +var exportFileExtToSuffix = map[string]string{ + ExportTypeDocx: ".docx", + ExportTypeXlsx: ".xlsx", + ExportTypePDF: ".pdf", +} + +// --- Feishu API response structures --- + +// apiResponse is the common Feishu API response wrapper. +type apiResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +// tokenResponse is the response for tenant_access_token API. +type tokenResponse struct { + apiResponse + TenantAccessToken string `json:"tenant_access_token"` + Expire int `json:"expire"` // seconds +} + +// wikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces. +type wikiSpaceListResponse struct { + apiResponse + Data struct { + Items []wikiSpace `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + } `json:"data"` +} + +// wikiSpace represents a Feishu Wiki space. +type wikiSpace struct { + SpaceID string `json:"space_id"` + Name string `json:"name"` + Description string `json:"description"` + Visibility string `json:"visibility"` // "public" or "private" +} + +// wikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes. +type wikiNodeListResponse struct { + apiResponse + Data struct { + Items []wikiNode `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + } `json:"data"` +} + +// wikiNode represents a node (document or folder) in a Feishu Wiki space. +type wikiNode struct { + SpaceID string `json:"space_id"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` // document token + ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides" + ParentNodeID string `json:"parent_node_id"` + NodeType string `json:"node_type"` // "origin" or "shortcut" + OriginNodeID string `json:"origin_node_id"` + OriginSpaceID string `json:"origin_space_id"` + HasChild bool `json:"has_child"` + Title string `json:"title"` + Creator string `json:"creator"` + Owner string `json:"owner"` + ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string) + ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes + NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string) + NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes +} + +// wikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node. +type wikiNodeInfoResponse struct { + apiResponse + Data struct { + Node wikiNode `json:"node"` + } `json:"data"` +} + +// --- Export task API responses --- + +// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content. +// Deprecated: prefer export API for full-fidelity document export. +type docRawContentResponse struct { + apiResponse + Data struct { + Content string `json:"content"` + } `json:"data"` +} + +// exportTaskCreateResponse is the response for POST /drive/v1/export_tasks. +type exportTaskCreateResponse struct { + apiResponse + Data struct { + Ticket string `json:"ticket"` + } `json:"data"` +} + +// exportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}. +type exportTaskStatusResponse struct { + apiResponse + Data struct { + Result struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + // JobStatus: 0=success, 1=initializing, 2=processing + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` + } `json:"result"` + } `json:"data"` +} + +// --- File download response --- + +// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes. +type driveFileMetaResponse struct { + apiResponse + Data struct { + Metas []struct { + DocToken string `json:"doc_token"` + DocType string `json:"doc_type"` + Title string `json:"title"` + } `json:"metas"` + } `json:"data"` +} + +// feishuCursor stores incremental sync state for Feishu. +type feishuCursor struct { + // LastSyncTime is the timestamp of the last successful sync. + LastSyncTime time.Time `json:"last_sync_time"` + + // SpaceNodeTimes maps space_id -> node_token -> last known edit time. + // Used to detect which nodes have changed since last sync. + SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"` +} diff --git a/internal/datasource/errors.go b/internal/datasource/errors.go new file mode 100644 index 00000000..7e0bb2c5 --- /dev/null +++ b/internal/datasource/errors.go @@ -0,0 +1,32 @@ +package datasource + +import "errors" + +// Error definitions for datasource operations +var ( + // Connector errors + ErrConnectorNil = errors.New("connector is nil") + ErrConnectorTypeEmpty = errors.New("connector type is empty") + ErrConnectorNotFound = errors.New("connector type not found in registry") + + // DataSource errors + ErrDataSourceNotFound = errors.New("data source not found") + ErrDataSourceInvalid = errors.New("data source configuration is invalid") + ErrDataSourceNotActive = errors.New("data source is not active") + + // Configuration errors + ErrInvalidConfig = errors.New("invalid configuration") + ErrInvalidCredentials = errors.New("invalid credentials") + + // Sync errors + ErrSyncFailed = errors.New("sync operation failed") + ErrSyncCanceled = errors.New("sync operation was canceled") + ErrFetchFailed = errors.New("failed to fetch items from source") + ErrResourceNotFound = errors.New("resource not found in source system") + + // Knowledge base errors + ErrKnowledgeBaseNotFound = errors.New("knowledge base not found") + + // Sync log errors + ErrSyncLogNotFound = errors.New("sync log not found") +) diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go new file mode 100644 index 00000000..96abe978 --- /dev/null +++ b/internal/datasource/scheduler.go @@ -0,0 +1,208 @@ +package datasource + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/robfig/cron/v3" +) + +// Scheduler manages cron-based periodic sync for data sources. +// +// robfig/cron fires at absolute wall-clock times (e.g. "0 0 * * * *" always fires +// at the top of every hour regardless of when the process started). So multiple +// instances will fire at the same moment. Dedup is handled by two layers: +// +// 1. HasRunningSync — if a previous sync is still running, skip (prevent overlap). +// 2. asynq.TaskID — deterministic ID per (dataSourceID, minute). Redis ensures +// only one task with a given ID is enqueued. Losers get ErrTaskIDConflict. +type Scheduler struct { + cron *cron.Cron + dsRepo interfaces.DataSourceRepository + syncLogRepo interfaces.SyncLogRepository + taskEnqueuer interfaces.TaskEnqueuer + + mu sync.Mutex + entries map[string]cron.EntryID // dataSourceID → cron entry ID +} + +// NewScheduler creates a new Scheduler. +func NewScheduler( + dsRepo interfaces.DataSourceRepository, + syncLogRepo interfaces.SyncLogRepository, + taskEnqueuer interfaces.TaskEnqueuer, +) *Scheduler { + return &Scheduler{ + cron: cron.New(cron.WithSeconds(), cron.WithChain( + cron.Recover(cron.DefaultLogger), + )), + dsRepo: dsRepo, + syncLogRepo: syncLogRepo, + taskEnqueuer: taskEnqueuer, + entries: make(map[string]cron.EntryID), + } +} + +// Start loads all active data sources from the database and registers their +// cron schedules. Then starts the cron runner in the background. +func (s *Scheduler) Start(ctx context.Context) error { + dataSources, err := s.dsRepo.FindActive(ctx) + if err != nil { + return fmt.Errorf("load active data sources: %w", err) + } + + for _, ds := range dataSources { + if ds.SyncSchedule == "" { + continue + } + if err := s.addEntry(ds); err != nil { + logger.Warnf(ctx, "[Scheduler] failed to register cron for ds=%s schedule=%q: %v", + ds.ID, ds.SyncSchedule, err) + } + } + + s.cron.Start() + logger.Infof(ctx, "[Scheduler] started with %d cron entries", len(s.entries)) + return nil +} + +// Stop gracefully stops the cron runner and waits for running jobs to finish. +func (s *Scheduler) Stop() { + ctx := s.cron.Stop() + <-ctx.Done() +} + +// AddOrUpdate registers (or re-registers) a cron entry for the given data source. +func (s *Scheduler) AddOrUpdate(ds *types.DataSource) error { + s.mu.Lock() + defer s.mu.Unlock() + + if entryID, ok := s.entries[ds.ID]; ok { + s.cron.Remove(entryID) + delete(s.entries, ds.ID) + } + + if ds.Status != types.DataSourceStatusActive || ds.SyncSchedule == "" { + return nil + } + + return s.addEntryLocked(ds) +} + +// Remove removes the cron entry for a data source. +func (s *Scheduler) Remove(dataSourceID string) { + s.mu.Lock() + defer s.mu.Unlock() + + if entryID, ok := s.entries[dataSourceID]; ok { + s.cron.Remove(entryID) + delete(s.entries, dataSourceID) + } +} + +func (s *Scheduler) addEntry(ds *types.DataSource) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.addEntryLocked(ds) +} + +func (s *Scheduler) addEntryLocked(ds *types.DataSource) error { + dsID := ds.ID + tenantID := ds.TenantID + + entryID, err := s.cron.AddFunc(ds.SyncSchedule, func() { + s.triggerSync(dsID, tenantID) + }) + if err != nil { + return fmt.Errorf("invalid cron expression %q: %w", ds.SyncSchedule, err) + } + + s.entries[dsID] = entryID + return nil +} + +// triggerSync is called by the cron runner on each tick. +// +// Layer 1 — DB: if a previous sync is still running, skip. This prevents +// overlap when a sync takes longer than the cron interval. +// +// Layer 2 — Redis: deterministic asynq.TaskID = "dssync::". +// Since robfig/cron fires at absolute wall-clock times, all instances trigger +// at the same minute. The first Enqueue wins; others get ErrTaskIDConflict. +func (s *Scheduler) triggerSync(dataSourceID string, tenantID uint64) { + ctx := context.Background() + + ds, err := s.dsRepo.FindByID(ctx, dataSourceID) + if err != nil || ds == nil || ds.Status != types.DataSourceStatusActive { + logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (not active or not found)", dataSourceID) + return + } + + // Layer 1: prevent overlap with a still-running sync + if running, _ := s.syncLogRepo.HasRunningSync(ctx, dataSourceID); running { + logger.Infof(ctx, "[Scheduler] skipping sync for ds=%s (previous sync still running)", dataSourceID) + return + } + + syncLog := &types.SyncLog{ + DataSourceID: dataSourceID, + TenantID: tenantID, + Status: types.SyncLogStatusRunning, + StartedAt: time.Now(), + } + if err := s.syncLogRepo.Create(ctx, syncLog); err != nil { + logger.Errorf(ctx, "[Scheduler] failed to create sync log for ds=%s: %v", dataSourceID, err) + return + } + + payload := &types.DataSourceSyncPayload{ + DataSourceID: dataSourceID, + TenantID: tenantID, + SyncLogID: syncLog.ID, + ForceFull: false, + } + payloadJSON, _ := json.Marshal(payload) + task := asynq.NewTask(types.TypeDataSourceSync, payloadJSON) + + // Layer 2: deterministic TaskID — all instances in the same minute produce the same ID + taskID := fmt.Sprintf("dssync:%s:%s", dataSourceID, time.Now().UTC().Truncate(time.Minute).Format("200601021504")) + + _, err = s.taskEnqueuer.Enqueue(task, + asynq.Queue("low"), + asynq.TaskID(taskID), + ) + if err != nil { + if err == asynq.ErrTaskIDConflict { + logger.Infof(ctx, "[Scheduler] sync already enqueued by another instance for ds=%s", dataSourceID) + syncLog.Status = types.SyncLogStatusCanceled + now := time.Now() + syncLog.FinishedAt = &now + syncLog.ErrorMessage = "deduplicated: another instance enqueued first" + _ = s.syncLogRepo.Update(ctx, syncLog) + return + } + logger.Errorf(ctx, "[Scheduler] failed to enqueue sync task for ds=%s: %v", dataSourceID, err) + syncLog.Status = types.SyncLogStatusFailed + now := time.Now() + syncLog.FinishedAt = &now + syncLog.ErrorMessage = fmt.Sprintf("enqueue failed: %v", err) + _ = s.syncLogRepo.Update(ctx, syncLog) + return + } + + logger.Infof(ctx, "[Scheduler] sync task enqueued for ds=%s syncLog=%s", dataSourceID, syncLog.ID) +} + +// EntryCount returns the number of active cron entries (for testing/monitoring). +func (s *Scheduler) EntryCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.entries) +} diff --git a/internal/datasource/scheduler_test.go b/internal/datasource/scheduler_test.go new file mode 100644 index 00000000..c27ed62a --- /dev/null +++ b/internal/datasource/scheduler_test.go @@ -0,0 +1,356 @@ +package datasource + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" +) + +// ────────────────────────────────────────────────────────────────────── +// Fake implementations for testing +// ────────────────────────────────────────────────────────────────────── + +// fakeDataSourceRepo is an in-memory DataSourceRepository. +type fakeDataSourceRepo struct { + mu sync.Mutex + dataSources map[string]*types.DataSource +} + +func newFakeDataSourceRepo() *fakeDataSourceRepo { + return &fakeDataSourceRepo{dataSources: make(map[string]*types.DataSource)} +} + +func (r *fakeDataSourceRepo) Create(_ context.Context, ds *types.DataSource) error { + r.mu.Lock() + defer r.mu.Unlock() + r.dataSources[ds.ID] = ds + return nil +} + +func (r *fakeDataSourceRepo) FindByID(_ context.Context, id string) (*types.DataSource, error) { + r.mu.Lock() + defer r.mu.Unlock() + ds, ok := r.dataSources[id] + if !ok { + return nil, ErrDataSourceNotFound + } + return ds, nil +} + +func (r *fakeDataSourceRepo) FindByKnowledgeBase(_ context.Context, kbID string) ([]*types.DataSource, error) { + return nil, nil +} + +func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) error { + r.mu.Lock() + defer r.mu.Unlock() + r.dataSources[ds.ID] = ds + return nil +} + +func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.dataSources, id) + return nil +} + +func (r *fakeDataSourceRepo) FindActive(_ context.Context) ([]*types.DataSource, error) { + r.mu.Lock() + defer r.mu.Unlock() + var result []*types.DataSource + for _, ds := range r.dataSources { + if ds.Status == types.DataSourceStatusActive && ds.SyncSchedule != "" { + result = append(result, ds) + } + } + return result, nil +} + +// fakeSyncLogRepo is an in-memory SyncLogRepository. +type fakeSyncLogRepo struct { + mu sync.Mutex + logs map[string]*types.SyncLog +} + +func newFakeSyncLogRepo() *fakeSyncLogRepo { + return &fakeSyncLogRepo{logs: make(map[string]*types.SyncLog)} +} + +func (r *fakeSyncLogRepo) Create(_ context.Context, log *types.SyncLog) error { + r.mu.Lock() + defer r.mu.Unlock() + if log.ID == "" { + log.ID = "log-" + time.Now().Format("150405.000") + } + r.logs[log.ID] = log + return nil +} + +func (r *fakeSyncLogRepo) FindByID(_ context.Context, id string) (*types.SyncLog, error) { + r.mu.Lock() + defer r.mu.Unlock() + l, ok := r.logs[id] + if !ok { + return nil, ErrSyncLogNotFound + } + return l, nil +} + +func (r *fakeSyncLogRepo) FindByDataSource(_ context.Context, dsID string, limit, offset int) ([]*types.SyncLog, error) { + return nil, nil +} + +func (r *fakeSyncLogRepo) FindLatest(_ context.Context, dsID string) (*types.SyncLog, error) { + return nil, nil +} + +func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error { + r.mu.Lock() + defer r.mu.Unlock() + r.logs[log.ID] = log + return nil +} + +func (r *fakeSyncLogRepo) CleanupOldLogs(_ context.Context, retentionDays int) error { + return nil +} + +func (r *fakeSyncLogRepo) HasRunningSync(_ context.Context, dsID string) (bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + for _, log := range r.logs { + if log.DataSourceID == dsID && log.Status == types.SyncLogStatusRunning { + return true, nil + } + } + return false, nil +} + +// fakeTaskEnqueuer counts how many tasks are enqueued. +type fakeTaskEnqueuer struct { + count atomic.Int64 +} + +func (e *fakeTaskEnqueuer) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) { + e.count.Add(1) + return &asynq.TaskInfo{ID: "task-fake"}, nil +} + +// ────────────────────────────────────────────────────────────────────── +// Tests +// ────────────────────────────────────────────────────────────────────── + +func TestScheduler_StartWithActiveDataSources(t *testing.T) { + repo := newFakeDataSourceRepo() + _ = repo.Create(context.Background(), &types.DataSource{ + ID: "ds-1", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "*/2 * * * * *", // every 2 seconds (6-field cron with seconds) + }) + _ = repo.Create(context.Background(), &types.DataSource{ + ID: "ds-2", + TenantID: 1, + Status: types.DataSourceStatusPaused, // should NOT be scheduled + SyncSchedule: "*/2 * * * * *", + }) + + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer) + + if err := scheduler.Start(context.Background()); err != nil { + t.Fatalf("Start() error: %v", err) + } + defer scheduler.Stop() + + // Only ds-1 should be registered (ds-2 is paused, not returned by FindActive) + if scheduler.EntryCount() != 1 { + t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount()) + } +} + +func TestScheduler_CronFires(t *testing.T) { + repo := newFakeDataSourceRepo() + _ = repo.Create(context.Background(), &types.DataSource{ + ID: "ds-fire", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "* * * * * *", // every second + }) + + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer) + + if err := scheduler.Start(context.Background()); err != nil { + t.Fatalf("Start() error: %v", err) + } + + // Wait for at least one tick + time.Sleep(2500 * time.Millisecond) + scheduler.Stop() + + if enqueuer.count.Load() == 0 { + t.Error("expected at least 1 enqueue, got 0") + } +} + +func TestScheduler_AddOrUpdate(t *testing.T) { + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer) + scheduler.cron.Start() + defer scheduler.Stop() + + ds := &types.DataSource{ + ID: "ds-new", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "0 0 * * * *", // every hour + } + + // Add + if err := scheduler.AddOrUpdate(ds); err != nil { + t.Fatalf("AddOrUpdate() error: %v", err) + } + if scheduler.EntryCount() != 1 { + t.Errorf("EntryCount() = %d, want 1", scheduler.EntryCount()) + } + + // Update schedule + ds.SyncSchedule = "0 30 * * * *" // every half hour + if err := scheduler.AddOrUpdate(ds); err != nil { + t.Fatalf("AddOrUpdate() (update) error: %v", err) + } + if scheduler.EntryCount() != 1 { + t.Errorf("after update: EntryCount() = %d, want 1 (should replace, not add)", scheduler.EntryCount()) + } +} + +func TestScheduler_AddOrUpdate_PausedIsNoop(t *testing.T) { + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer) + scheduler.cron.Start() + defer scheduler.Stop() + + ds := &types.DataSource{ + ID: "ds-paused", + TenantID: 1, + Status: types.DataSourceStatusPaused, + SyncSchedule: "0 0 * * * *", + } + + if err := scheduler.AddOrUpdate(ds); err != nil { + t.Fatalf("AddOrUpdate() error: %v", err) + } + if scheduler.EntryCount() != 0 { + t.Errorf("paused ds should not be scheduled, EntryCount() = %d", scheduler.EntryCount()) + } +} + +func TestScheduler_AddOrUpdate_EmptyScheduleIsNoop(t *testing.T) { + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer) + scheduler.cron.Start() + defer scheduler.Stop() + + ds := &types.DataSource{ + ID: "ds-no-sched", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "", + } + + if err := scheduler.AddOrUpdate(ds); err != nil { + t.Fatalf("AddOrUpdate() error: %v", err) + } + if scheduler.EntryCount() != 0 { + t.Errorf("empty schedule should not be registered, EntryCount() = %d", scheduler.EntryCount()) + } +} + +func TestScheduler_Remove(t *testing.T) { + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer) + scheduler.cron.Start() + defer scheduler.Stop() + + ds := &types.DataSource{ + ID: "ds-rm", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "0 0 * * * *", + } + + _ = scheduler.AddOrUpdate(ds) + if scheduler.EntryCount() != 1 { + t.Fatalf("pre-remove: EntryCount() = %d, want 1", scheduler.EntryCount()) + } + + scheduler.Remove("ds-rm") + if scheduler.EntryCount() != 0 { + t.Errorf("post-remove: EntryCount() = %d, want 0", scheduler.EntryCount()) + } + + // Remove non-existent is safe + scheduler.Remove("does-not-exist") +} + +func TestScheduler_InvalidCron(t *testing.T) { + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(newFakeDataSourceRepo(), newFakeSyncLogRepo(), enqueuer) + scheduler.cron.Start() + defer scheduler.Stop() + + ds := &types.DataSource{ + ID: "ds-bad", + TenantID: 1, + Status: types.DataSourceStatusActive, + SyncSchedule: "not a cron", + } + + err := scheduler.AddOrUpdate(ds) + if err == nil { + t.Fatal("expected error for invalid cron expression") + } + if scheduler.EntryCount() != 0 { + t.Errorf("invalid cron should not be registered, EntryCount() = %d", scheduler.EntryCount()) + } +} + +func TestScheduler_TriggerSync_InactiveSkipped(t *testing.T) { + repo := newFakeDataSourceRepo() + // Create a data source that is paused + _ = repo.Create(context.Background(), &types.DataSource{ + ID: "ds-inactive", + TenantID: 1, + Status: types.DataSourceStatusPaused, + }) + + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer) + + // Directly call triggerSync — it should skip because ds is not active + scheduler.triggerSync("ds-inactive", 1) + + if enqueuer.count.Load() != 0 { + t.Error("should not enqueue for inactive data source") + } +} + +func TestScheduler_TriggerSync_NotFound(t *testing.T) { + repo := newFakeDataSourceRepo() + enqueuer := &fakeTaskEnqueuer{} + scheduler := NewScheduler(repo, newFakeSyncLogRepo(), enqueuer) + + // Should not panic, just skip + scheduler.triggerSync("nonexistent", 1) + + if enqueuer.count.Load() != 0 { + t.Error("should not enqueue for non-existent data source") + } +} diff --git a/internal/handler/datasource.go b/internal/handler/datasource.go new file mode 100644 index 00000000..418be481 --- /dev/null +++ b/internal/handler/datasource.go @@ -0,0 +1,549 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// DataSourceHandler handles HTTP requests for data source management +type DataSourceHandler struct { + service interfaces.DataSourceService +} + +// NewDataSourceHandler creates a new data source handler +func NewDataSourceHandler(service interfaces.DataSourceService) *DataSourceHandler { + return &DataSourceHandler{ + service: service, + } +} + +// getTenantID safely extracts and validates tenant ID from context +// Returns 0 if tenant ID is not found (caller should return 401) +func (h *DataSourceHandler) getTenantID(c *gin.Context) uint64 { + tenantID := c.GetUint64(types.TenantIDContextKey.String()) + return tenantID +} + +// CreateDataSource godoc +// @Summary Create a new data source +// @Description Create a new data source configuration for a knowledge base +// @Tags DataSource +// @Accept json +// @Produce json +// @Param request body types.DataSource true "Data source configuration" +// @Success 201 {object} types.DataSource +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource [post] +func (h *DataSourceHandler) CreateDataSource(c *gin.Context) { + ctx := c.Request.Context() + + // Extract tenant ID from context (set by auth middleware) + tenantID := c.GetUint64(types.TenantIDContextKey.String()) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: tenant context missing"}) + return + } + + var req types.DataSource + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + + // Enforce tenant isolation + req.TenantID = tenantID + + ds, err := h.service.CreateDataSource(ctx, &req) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusCreated, ds) +} + +// GetDataSource godoc +// @Summary Get a data source by ID +// @Description Retrieve a data source configuration by ID +// @Tags DataSource +// @Produce json +// @Param id path string true "Data source ID" +// @Success 200 {object} types.DataSource +// @Failure 404 {object} map[string]string +// @Router /api/v1/datasource/{id} [get] +func (h *DataSourceHandler) GetDataSource(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + // Verify ownership (data source belongs to this tenant) + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + c.JSON(http.StatusOK, ds) +} + +// ListDataSources godoc +// @Summary List data sources for a knowledge base +// @Description List all data sources for a specific knowledge base +// @Tags DataSource +// @Produce json +// @Param kb_id query string true "Knowledge base ID" +// @Success 200 {object} []types.DataSource +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource [get] +func (h *DataSourceHandler) ListDataSources(c *gin.Context) { + ctx := c.Request.Context() + + // Extract tenant ID from context + tenantID := c.GetUint64(types.TenantIDContextKey.String()) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: tenant context missing"}) + return + } + + kbID := c.Query("kb_id") + if kbID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "kb_id is required"}) + return + } + + dataSources, err := h.service.ListDataSources(ctx, kbID) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list data sources"}) + return + } + + if dataSources == nil { + dataSources = make([]*types.DataSource, 0) + } + c.JSON(http.StatusOK, dataSources) +} + +// UpdateDataSource godoc +// @Summary Update a data source +// @Description Update an existing data source configuration +// @Tags DataSource +// @Accept json +// @Produce json +// @Param id path string true "Data source ID" +// @Param request body types.DataSource true "Updated configuration" +// @Success 200 {object} types.DataSource +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id} [put] +func (h *DataSourceHandler) UpdateDataSource(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + var req types.DataSource + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + + // Fetch existing data source to verify ownership + existing, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if existing.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + req.ID = id + req.TenantID = tenantID + ds, err := h.service.UpdateDataSource(ctx, &req) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, ds) +} + +// DeleteDataSource godoc +// @Summary Delete a data source +// @Description Delete a data source (soft delete) +// @Tags DataSource +// @Param id path string true "Data source ID" +// @Success 204 +// @Failure 404 {object} map[string]string +// @Router /api/v1/datasource/{id} [delete] +func (h *DataSourceHandler) DeleteDataSource(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership before deleting + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + if err := h.service.DeleteDataSource(ctx, id); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete data source"}) + return + } + + c.Status(http.StatusNoContent) +} + +// ValidateConnection godoc +// @Summary Test data source connection +// @Description Validate the connection to an external data source +// @Tags DataSource +// @Param id path string true "Data source ID" +// @Success 200 {object} map[string]string +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/validate [post] +func (h *DataSourceHandler) ValidateConnection(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + if err := h.service.ValidateConnection(ctx, id); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "connected"}) +} + +// ValidateCredentials godoc +// @Summary Test connection with raw credentials (no persistence) +// @Description Validate connectivity to an external data source using type + credentials +// +// without creating or updating any database records. +// Used by the frontend "Test Connection" button during data source creation. +// +// @Tags DataSource +// @Accept json +// @Produce json +// @Param request body object true "type and credentials" +// @Success 200 {object} map[string]string +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/validate-credentials [post] +func (h *DataSourceHandler) ValidateCredentials(c *gin.Context) { + ctx := c.Request.Context() + + var req struct { + Type string `json:"type" binding:"required"` + Credentials map[string]interface{} `json:"credentials" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request: type and credentials are required"}) + return + } + + if err := h.service.ValidateCredentials(ctx, req.Type, req.Credentials); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "connected"}) +} +// @Summary List available resources in data source +// @Description List resources available for sync in the external system +// @Tags DataSource +// @Produce json +// @Param id path string true "Data source ID" +// @Success 200 {object} []types.Resource +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/resources [get] +func (h *DataSourceHandler) ListAvailableResources(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + resources, err := h.service.ListAvailableResources(ctx, id) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if resources == nil { + resources = make([]types.Resource, 0) + } + c.JSON(http.StatusOK, resources) +} + +// ManualSync godoc +// @Summary Trigger immediate sync +// @Description Trigger an immediate sync for a data source +// @Tags DataSource +// @Param id path string true "Data source ID" +// @Success 200 {object} types.SyncLog +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/sync [post] +func (h *DataSourceHandler) ManualSync(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + syncLog, err := h.service.ManualSync(ctx, id) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, syncLog) +} + +// PauseDataSource godoc +// @Summary Pause data source +// @Description Pause a data source's scheduled syncs +// @Tags DataSource +// @Param id path string true "Data source ID" +// @Success 200 {object} map[string]string +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/pause [post] +func (h *DataSourceHandler) PauseDataSource(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + if err := h.service.PauseDataSource(ctx, id); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "paused"}) +} + +// ResumeDataSource godoc +// @Summary Resume data source +// @Description Resume a paused data source +// @Tags DataSource +// @Param id path string true "Data source ID" +// @Success 200 {object} map[string]string +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/resume [post] +func (h *DataSourceHandler) ResumeDataSource(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + if err := h.service.ResumeDataSource(ctx, id); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"status": "active"}) +} + +// GetSyncLogs godoc +// @Summary Get sync logs +// @Description Retrieve sync history for a data source +// @Tags DataSource +// @Produce json +// @Param id path string true "Data source ID" +// @Param limit query int false "Limit (default: 10)" +// @Param offset query int false "Offset (default: 0)" +// @Success 200 {object} []types.SyncLog +// @Failure 400 {object} map[string]string +// @Router /api/v1/datasource/{id}/logs [get] +func (h *DataSourceHandler) GetSyncLogs(c *gin.Context) { + ctx := c.Request.Context() + tenantID := h.getTenantID(c) + if tenantID == 0 { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + + id := c.Param("id") + + // Verify ownership + ds, err := h.service.GetDataSource(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "data source not found"}) + return + } + + if ds.TenantID != tenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "access denied"}) + return + } + + limit := 10 + offset := 0 + + if l := c.Query("limit"); l != "" { + if v, err := strconv.Atoi(l); err == nil && v > 0 { + limit = v + } + } + + if o := c.Query("offset"); o != "" { + if v, err := strconv.Atoi(o); err == nil && v >= 0 { + offset = v + } + } + + logs, err := h.service.GetSyncLogs(ctx, id, limit, offset) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if logs == nil { + logs = make([]*types.SyncLog, 0) + } + c.JSON(http.StatusOK, logs) +} + +// GetSyncLog godoc +// @Summary Get specific sync log +// @Description Retrieve a specific sync log entry +// @Tags DataSource +// @Produce json +// @Param log_id path string true "Sync log ID" +// @Success 200 {object} types.SyncLog +// @Failure 404 {object} map[string]string +// @Router /api/v1/datasource/logs/{log_id} [get] +func (h *DataSourceHandler) GetSyncLog(c *gin.Context) { + ctx := c.Request.Context() + logID := c.Param("log_id") + + log, err := h.service.GetSyncLog(ctx, logID) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "sync log not found"}) + return + } + + c.JSON(http.StatusOK, log) +} + +// GetAvailableConnectors godoc +// @Summary Get available connectors +// @Description Get list of available data source connectors +// @Tags DataSource +// @Produce json +// @Success 200 {object} []datasource.ConnectorMetadata +// @Router /api/v1/datasource/types [get] +func (h *DataSourceHandler) GetAvailableConnectors(c *gin.Context) { + connectors := datasource.ListAvailableConnectors() + c.JSON(http.StatusOK, connectors) +} diff --git a/internal/router/router.go b/internal/router/router.go index 5ef99bde..67df9dd9 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -60,6 +60,7 @@ type RouterParams struct { SkillHandler *handler.SkillHandler OrganizationHandler *handler.OrganizationHandler IMHandler *handler.IMHandler + DataSourceHandler *handler.DataSourceHandler } // NewRouter 创建新的路由 @@ -140,6 +141,7 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterSkillRoutes(v1, params.SkillHandler) RegisterOrganizationRoutes(v1, params.OrganizationHandler) RegisterIMChannelRoutes(v1, params.IMHandler) + RegisterDataSourceRoutes(v1, params.DataSourceHandler) } return r @@ -740,3 +742,36 @@ func serveFiles(r *gin.Engine) { } }) } + +// RegisterDataSourceRoutes 注册数据源相关的路由 +func RegisterDataSourceRoutes(r *gin.RouterGroup, handler *handler.DataSourceHandler) { + // Data source routes + ds := r.Group("/datasource") + { + // Get available connector types + ds.GET("/types", handler.GetAvailableConnectors) + + // Validate credentials without persistence (for "Test Connection" button) + ds.POST("/validate-credentials", handler.ValidateCredentials) + + // CRUD operations + ds.POST("", handler.CreateDataSource) + ds.GET("", handler.ListDataSources) + ds.GET("/:id", handler.GetDataSource) + ds.PUT("/:id", handler.UpdateDataSource) + ds.DELETE("/:id", handler.DeleteDataSource) + + // Connection and resource management + ds.POST("/:id/validate", handler.ValidateConnection) + ds.GET("/:id/resources", handler.ListAvailableResources) + + // Sync management + ds.POST("/:id/sync", handler.ManualSync) + ds.POST("/:id/pause", handler.PauseDataSource) + ds.POST("/:id/resume", handler.ResumeDataSource) + + // Sync logs + ds.GET("/:id/logs", handler.GetSyncLogs) + ds.GET("/logs/:log_id", handler.GetSyncLog) + } +} diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go index 45b4c43e..de851c70 100644 --- a/internal/router/sync_task.go +++ b/internal/router/sync_task.go @@ -75,6 +75,7 @@ type SyncTaskParams struct { KnowledgeService interfaces.KnowledgeService KnowledgeBaseService interfaces.KnowledgeBaseService TagService interfaces.KnowledgeTagService + DataSourceService interfaces.DataSourceService ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"` DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"` ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"` @@ -96,5 +97,6 @@ func RegisterSyncHandlers(params SyncTaskParams) { params.Executor.RegisterHandler(types.TypeIndexDelete, params.TagService.ProcessIndexDelete) params.Executor.RegisterHandler(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete) params.Executor.RegisterHandler(types.TypeImageMultimodal, params.ImageMultimodal.Handle) + params.Executor.RegisterHandler(types.TypeDataSourceSync, params.DataSourceService.ProcessSync) logger.Infof(context.Background(), "[SyncTask] All task handlers registered (Lite mode, no Redis)") } diff --git a/internal/router/task.go b/internal/router/task.go index 39c45a2d..ff4e99fe 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -19,6 +19,7 @@ type AsynqTaskParams struct { KnowledgeService interfaces.KnowledgeService KnowledgeBaseService interfaces.KnowledgeBaseService TagService interfaces.KnowledgeTagService + DataSourceService interfaces.DataSourceService ChunkExtractor interfaces.TaskHandler `name:"chunkExtractor"` DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"` ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"` @@ -108,6 +109,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register image multimodal handler mux.HandleFunc(types.TypeImageMultimodal, params.ImageMultimodal.Handle) + // Register data source sync handler + mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync) + go func() { // Start the server if err := params.Server.Run(mux); err != nil { diff --git a/internal/types/datasource.go b/internal/types/datasource.go new file mode 100644 index 00000000..12667d2b --- /dev/null +++ b/internal/types/datasource.go @@ -0,0 +1,413 @@ +package types + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// Data source types and constants +const ( + // Connector types + ConnectorTypeFeishu = "feishu" + ConnectorTypeNotion = "notion" + ConnectorTypeConfluence = "confluence" + ConnectorTypeYuque = "yuque" + ConnectorTypeGitHub = "github" + ConnectorTypeGoogleDrive = "google_drive" + ConnectorTypeOneDrive = "onedrive" + ConnectorTypeDingTalk = "dingtalk" + ConnectorTypeWecomDoc = "wecom_doc" + ConnectorTypeWebCrawler = "web_crawler" + ConnectorTypeSlack = "slack" + ConnectorTypeIMAP = "imap" + ConnectorTypeRSS = "rss" + + // Sync modes + SyncModeIncremental = "incremental" + SyncModeFull = "full" + + // Data source status + DataSourceStatusActive = "active" + DataSourceStatusPaused = "paused" + DataSourceStatusError = "error" + DataSourceStatusDeleted = "deleted" + + // Sync log status + SyncLogStatusRunning = "running" + SyncLogStatusSuccess = "success" + SyncLogStatusPartial = "partial" + SyncLogStatusFailed = "failed" + SyncLogStatusCanceled = "canceled" + + // Conflict resolution strategies + ConflictStrategyOverwrite = "overwrite" + ConflictStrategySkip = "skip" +) + +// DataSource represents a configured external data source for synchronization +type DataSource struct { + // Unique identifier + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + + // Tenant ID for multi-tenancy + TenantID uint64 `json:"tenant_id" gorm:"index"` + + // Target knowledge base ID + KnowledgeBaseID string `json:"knowledge_base_id" gorm:"index"` + + // User-friendly name + Name string `json:"name"` + + // Connector type (feishu, notion, confluence, etc.) + Type string `json:"type" gorm:"type:varchar(50);index"` + + // Encrypted configuration (API credentials, tokens, etc.) + // Stored as JSON with AES-256-GCM encryption + Config JSON `json:"config" gorm:"type:jsonb"` + + // Cron expression for scheduled syncs (e.g., "0 */6 * * *" = every 6 hours) + SyncSchedule string `json:"sync_schedule"` + + // Sync mode: "incremental" (recommended) or "full" + SyncMode string `json:"sync_mode" gorm:"type:varchar(20);default:'incremental'"` + + // Current status: active, paused, error + Status string `json:"status" gorm:"type:varchar(32);default:'active'"` + + // Conflict resolution strategy: overwrite or skip + ConflictStrategy string `json:"conflict_strategy" gorm:"type:varchar(32);default:'overwrite'"` + + // Whether to sync deletions from source + SyncDeletions bool `json:"sync_deletions" gorm:"default:true"` + + // Last successful sync timestamp + LastSyncAt *time.Time `json:"last_sync_at"` + + // Cursor or state for incremental sync (connector-specific) + LastSyncCursor JSON `json:"last_sync_cursor" gorm:"type:jsonb"` + + // Summary of last sync result + LastSyncResult JSON `json:"last_sync_result" gorm:"type:jsonb"` + + // Error message if status is "error" + ErrorMessage string `json:"error_message"` + + // Number of days to keep sync logs (default: 30) + SyncLogRetentionDays int `json:"sync_log_retention_days" gorm:"default:30"` + + // Creation timestamp + CreatedAt time.Time `json:"created_at"` + + // Last update timestamp + UpdatedAt time.Time `json:"updated_at"` + + // Soft delete timestamp + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` + + // Total items synced (not stored in DB, calculated on query) + TotalItemsSynced int64 `json:"total_items_synced" gorm:"-"` + + // Latest sync log (not stored in DB, populated on query) + LatestSyncLog *SyncLog `json:"latest_sync_log" gorm:"-"` +} + +// TableName specifies the table name for DataSource +func (d *DataSource) TableName() string { + return "data_sources" +} + +// BeforeCreate hook to generate UUID +func (d *DataSource) BeforeCreate(tx *gorm.DB) error { + if d.ID == "" { + d.ID = uuid.New().String() + } + return nil +} + +// SyncLog records the execution of a sync task +type SyncLog struct { + // Unique identifier + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + + // Reference to the data source + DataSourceID string `json:"data_source_id" gorm:"index"` + + // Tenant ID + TenantID uint64 `json:"tenant_id" gorm:"index"` + + // Sync status: running, success, partial, failed, canceled + Status string `json:"status" gorm:"type:varchar(32);index"` + + // Sync start time + StartedAt time.Time `json:"started_at"` + + // Sync completion time + FinishedAt *time.Time `json:"finished_at"` + + // Total items fetched from source + ItemsTotal int `json:"items_total"` + + // New items created in knowledge base + ItemsCreated int `json:"items_created"` + + // Existing items updated + ItemsUpdated int `json:"items_updated"` + + // Items deleted from knowledge base + ItemsDeleted int `json:"items_deleted"` + + // Items skipped (no changes detected) + ItemsSkipped int `json:"items_skipped"` + + // Items that failed to sync + ItemsFailed int `json:"items_failed"` + + // Error details if status is "failed" + ErrorMessage string `json:"error_message"` + + // Detailed sync result (JSON-encoded) + Result JSON `json:"result" gorm:"type:jsonb"` + + // Creation timestamp (usually same as StartedAt) + CreatedAt time.Time `json:"created_at"` + + // Last update timestamp + UpdatedAt time.Time `json:"updated_at"` +} + +// TableName specifies the table name for SyncLog +func (s *SyncLog) TableName() string { + return "sync_logs" +} + +// BeforeCreate hook to generate UUID +func (s *SyncLog) BeforeCreate(tx *gorm.DB) error { + if s.ID == "" { + s.ID = uuid.New().String() + } + if s.StartedAt.IsZero() { + s.StartedAt = time.Now().UTC() + } + return nil +} + +// DataSourceConfig represents the unencrypted configuration structure +// Each connector type will have its own specific fields +type DataSourceConfig struct { + // Common fields applicable to most connectors + Type string `json:"type"` + + // OAuth/API credentials (varies by connector) + Credentials map[string]interface{} `json:"credentials"` + + // Selected resource IDs to sync (e.g., folder IDs, space IDs) + ResourceIDs []string `json:"resource_ids"` + + // Connector-specific configuration + Settings map[string]interface{} `json:"settings"` +} + +// Resource represents a syncable resource (document, folder, space) from external system +type Resource struct { + // Unique identifier in the external system + ExternalID string `json:"external_id"` + + // Display name + Name string `json:"name"` + + // Resource type (document, folder, space, page, etc.) + Type string `json:"type"` + + // Optional description + Description string `json:"description"` + + // URL to access in external system + URL string `json:"url"` + + // Last modified time in external system + ModifiedAt time.Time `json:"modified_at"` + + // For hierarchical resources (parent ID if applicable) + ParentID string `json:"parent_id,omitempty"` + + // Additional metadata + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// FetchedItem represents a single document/content item fetched from external source +type FetchedItem struct { + // Unique ID in the external system + ExternalID string `json:"external_id"` + + // Title of the content + Title string `json:"title"` + + // Content in bytes (Markdown format preferred) + Content []byte `json:"content"` + + // MIME type (text/markdown, text/html, application/pdf, etc.) + ContentType string `json:"content_type"` + + // Suggested file name + FileName string `json:"file_name"` + + // Original URL in external system + URL string `json:"url"` + + // When last modified in external system + UpdatedAt time.Time `json:"updated_at"` + + // Additional metadata to preserve + Metadata map[string]string `json:"metadata"` + + // Whether the item was deleted in the source + IsDeleted bool `json:"is_deleted"` + + // Source resource ID (e.g., folder ID this document belongs to) + SourceResourceID string `json:"source_resource_id"` +} + +// SyncCursor represents the position/state for incremental sync +// Connector-specific structure allows flexibility +type SyncCursor struct { + // Timestamp of last sync + LastSyncTime time.Time `json:"last_sync_time"` + + // Connector-specific cursor (e.g., pagination token, offset, etc.) + ConnectorCursor map[string]interface{} `json:"connector_cursor"` + + // Hash of the last full sync to detect schema changes + LastSchemaHash string `json:"last_schema_hash"` +} + +// SyncResult summarizes the outcome of a sync operation +type SyncResult struct { + // Total items processed + Total int `json:"total"` + + // Items created + Created int `json:"created"` + + // Items updated + Updated int `json:"updated"` + + // Items deleted + Deleted int `json:"deleted"` + + // Items skipped (no changes) + Skipped int `json:"skipped"` + + // Items that failed + Failed int `json:"failed"` + + // Detailed error messages + Errors []string `json:"errors,omitempty"` + + // Updated cursor for next incremental sync + NextCursor *SyncCursor `json:"next_cursor,omitempty"` +} + +// DataSourceSyncPayload represents the asynq task payload for data source sync +type DataSourceSyncPayload struct { + // Data source ID to sync + DataSourceID string `json:"data_source_id"` + + // Tenant ID + TenantID uint64 `json:"tenant_id"` + + // Sync log ID (for tracking) + SyncLogID string `json:"sync_log_id"` + + // Force full sync even if incremental mode is configured + ForceFull bool `json:"force_full"` + + // Maximum number of items to fetch (0 = unlimited) + MaxItems int `json:"max_items,omitempty"` +} + +// ToJSON converts a value to JSON type +func (d *DataSourceConfig) ToJSON() (JSON, error) { + if d == nil { + return nil, nil + } + bytes, err := json.Marshal(d) + if err != nil { + return nil, err + } + return JSON(bytes), nil +} + +// ToJSON converts a value to JSON type +func (r *SyncCursor) ToJSON() (JSON, error) { + if r == nil { + return nil, nil + } + bytes, err := json.Marshal(r) + if err != nil { + return nil, err + } + return JSON(bytes), nil +} + +// ToJSON converts a value to JSON type +func (r *SyncResult) ToJSON() (JSON, error) { + if r == nil { + return nil, nil + } + bytes, err := json.Marshal(r) + if err != nil { + return nil, err + } + return JSON(bytes), nil +} + +// ParseConfig parses the encrypted config JSON back to DataSourceConfig +func (d *DataSource) ParseConfig() (*DataSourceConfig, error) { + if len(d.Config) == 0 { + return nil, nil + } + var config DataSourceConfig + if err := json.Unmarshal(d.Config, &config); err != nil { + return nil, err + } + return &config, nil +} + +// ParseSyncCursor parses the cursor JSON +func (d *DataSource) ParseSyncCursor() (*SyncCursor, error) { + if len(d.LastSyncCursor) == 0 { + return nil, nil + } + var cursor SyncCursor + if err := json.Unmarshal(d.LastSyncCursor, &cursor); err != nil { + return nil, err + } + return &cursor, nil +} + +// ParseSyncResult parses the result JSON +func (d *DataSource) ParseSyncResult() (*SyncResult, error) { + if len(d.LastSyncResult) == 0 { + return nil, nil + } + var result SyncResult + if err := json.Unmarshal(d.LastSyncResult, &result); err != nil { + return nil, err + } + return &result, nil +} + +// ParseSyncLogResult parses the result JSON from sync log +func (s *SyncLog) ParseResult() (*SyncResult, error) { + if len(s.Result) == 0 { + return nil, nil + } + var result SyncResult + if err := json.Unmarshal(s.Result, &result); err != nil { + return nil, err + } + return &result, nil +} diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index 54b9b31e..a08df669 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -14,6 +14,7 @@ const ( TypeDataTableSummary = "datatable:summary" // 表格摘要任务 TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption) TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引) + TypeDataSourceSync = "datasource:sync" // 数据源同步任务 ) // ExtractChunkPayload represents the extract chunk task payload diff --git a/internal/types/interfaces/datasource.go b/internal/types/interfaces/datasource.go new file mode 100644 index 00000000..71172c88 --- /dev/null +++ b/internal/types/interfaces/datasource.go @@ -0,0 +1,100 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" +) + +// DataSourceService defines the interface for data source management operations +type DataSourceService interface { + // CreateDataSource creates a new data source configuration + CreateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) + + // GetDataSource retrieves a data source by ID + GetDataSource(ctx context.Context, id string) (*types.DataSource, error) + + // ListDataSources lists all data sources for a knowledge base + ListDataSources(ctx context.Context, kbID string) ([]*types.DataSource, error) + + // UpdateDataSource updates an existing data source + UpdateDataSource(ctx context.Context, ds *types.DataSource) (*types.DataSource, error) + + // DeleteDataSource deletes a data source (soft delete) + DeleteDataSource(ctx context.Context, id string) error + + // ValidateConnection tests the connection to an external data source + ValidateConnection(ctx context.Context, dsID string) error + + // ValidateCredentials tests connectivity using raw credentials without persisting anything. + // This is used by the frontend "Test Connection" button before creating a data source. + ValidateCredentials(ctx context.Context, connectorType string, credentials map[string]interface{}) error + + // ListAvailableResources lists resources available for sync in the external system + ListAvailableResources(ctx context.Context, dsID string) ([]types.Resource, error) + + // ManualSync triggers an immediate sync for a data source + ManualSync(ctx context.Context, dsID string) (*types.SyncLog, error) + + // PauseDataSource pauses a data source's scheduled syncs + PauseDataSource(ctx context.Context, id string) error + + // ResumeDataSource resumes a paused data source + ResumeDataSource(ctx context.Context, id string) error + + // GetSyncLogs retrieves sync history for a data source + GetSyncLogs(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) + + // GetSyncLog retrieves a specific sync log entry + GetSyncLog(ctx context.Context, syncLogID string) (*types.SyncLog, error) + + // ProcessSync handles the actual sync operation (called by asynq task) + ProcessSync(ctx context.Context, task *asynq.Task) error +} + +// DataSourceRepository defines database access patterns for data sources +type DataSourceRepository interface { + // Create inserts a new data source record + Create(ctx context.Context, ds *types.DataSource) error + + // FindByID retrieves a data source by ID + FindByID(ctx context.Context, id string) (*types.DataSource, error) + + // FindByKnowledgeBase lists all data sources for a knowledge base + FindByKnowledgeBase(ctx context.Context, kbID string) ([]*types.DataSource, error) + + // Update updates an existing data source + Update(ctx context.Context, ds *types.DataSource) error + + // Delete performs a soft delete + Delete(ctx context.Context, id string) error + + // FindActive retrieves all active data sources (used for scheduling) + FindActive(ctx context.Context) ([]*types.DataSource, error) +} + +// SyncLogRepository defines database access patterns for sync logs +type SyncLogRepository interface { + // Create inserts a new sync log entry + Create(ctx context.Context, log *types.SyncLog) error + + // FindByID retrieves a sync log by ID + FindByID(ctx context.Context, id string) (*types.SyncLog, error) + + // FindByDataSource lists sync logs for a data source with pagination + FindByDataSource(ctx context.Context, dsID string, limit int, offset int) ([]*types.SyncLog, error) + + // FindLatest retrieves the most recent sync log for a data source + FindLatest(ctx context.Context, dsID string) (*types.SyncLog, error) + + // HasRunningSync checks if a data source has any sync currently in "running" status. + // Used to prevent overlapping sync executions. + HasRunningSync(ctx context.Context, dsID string) (bool, error) + + // Update updates an existing sync log entry + Update(ctx context.Context, log *types.SyncLog) error + + // CleanupOldLogs deletes sync logs older than the retention period + CleanupOldLogs(ctx context.Context, retentionDays int) error +} diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index 1c824590..ab4459b1 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -209,6 +209,10 @@ type KnowledgeRepository interface { // SearchKnowledge searches knowledge items by keyword across the tenant. // fileTypes: optional list of file extensions to filter by (e.g., ["csv", "xlsx"]) SearchKnowledge(ctx context.Context, tenantID uint64, keyword string, offset, limit int, fileTypes []string) ([]*types.Knowledge, bool, error) + + // FindByMetadataKey finds a knowledge item by a key-value pair in the metadata JSON column. + // Used by data source sync to locate existing items by external_id. + FindByMetadataKey(ctx context.Context, tenantID uint64, kbID string, key string, value string) (*types.Knowledge, error) // SearchKnowledgeInScopes searches knowledge items by keyword within the given (tenant_id, kb_id) scopes (own + shared). SearchKnowledgeInScopes(ctx context.Context, scopes []types.KnowledgeSearchScope, keyword string, offset, limit int, fileTypes []string) ([]*types.Knowledge, bool, error) // ListIDsByTagID returns all knowledge IDs that have the specified tag ID. diff --git a/migrations/versioned/000028_datasource_tables.down.sql b/migrations/versioned/000028_datasource_tables.down.sql new file mode 100644 index 00000000..af337490 --- /dev/null +++ b/migrations/versioned/000028_datasource_tables.down.sql @@ -0,0 +1,6 @@ +-- Rollback: 000028_datasource_tables +DROP TRIGGER IF EXISTS trg_sync_logs_updated_at ON sync_logs; +DROP TRIGGER IF EXISTS trg_data_sources_updated_at ON data_sources; + +DROP TABLE IF EXISTS sync_logs; +DROP TABLE IF EXISTS data_sources; diff --git a/migrations/versioned/000028_datasource_tables.up.sql b/migrations/versioned/000028_datasource_tables.up.sql new file mode 100644 index 00000000..2f671158 --- /dev/null +++ b/migrations/versioned/000028_datasource_tables.up.sql @@ -0,0 +1,78 @@ +-- Migration: 000028_datasource_tables +-- Description: Create data_sources and sync_logs tables for external data source management +DO $$ BEGIN RAISE NOTICE '[Migration 000028] Creating data_sources and sync_logs tables'; END $$; + +-- Create data_sources table for managing external data source configurations +CREATE TABLE IF NOT EXISTS data_sources ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + tenant_id BIGINT NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL, + config JSONB, + sync_schedule VARCHAR(100), + sync_mode VARCHAR(20) DEFAULT 'incremental', + status VARCHAR(32) DEFAULT 'active', + conflict_strategy VARCHAR(32) DEFAULT 'overwrite', + sync_deletions BOOLEAN DEFAULT true, + last_sync_at TIMESTAMP NULL, + last_sync_cursor JSONB, + last_sync_result JSONB, + error_message TEXT, + sync_log_retention_days INT DEFAULT 30, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL +); + +CREATE INDEX IF NOT EXISTS idx_data_sources_tenant_id ON data_sources (tenant_id); +CREATE INDEX IF NOT EXISTS idx_data_sources_knowledge_base_id ON data_sources (knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_data_sources_type ON data_sources (type); +CREATE INDEX IF NOT EXISTS idx_data_sources_status ON data_sources (status); +CREATE INDEX IF NOT EXISTS idx_data_sources_deleted_at ON data_sources (deleted_at); + +-- Create sync_logs table for tracking sync operation history +CREATE TABLE IF NOT EXISTS sync_logs ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + data_source_id VARCHAR(36) NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE, + tenant_id BIGINT NOT NULL, + status VARCHAR(32) NOT NULL, + started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + finished_at TIMESTAMP NULL, + items_total INT DEFAULT 0, + items_created INT DEFAULT 0, + items_updated INT DEFAULT 0, + items_deleted INT DEFAULT 0, + items_skipped INT DEFAULT 0, + items_failed INT DEFAULT 0, + error_message TEXT, + result JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_sync_logs_data_source_id ON sync_logs (data_source_id); +CREATE INDEX IF NOT EXISTS idx_sync_logs_tenant_id ON sync_logs (tenant_id); +CREATE INDEX IF NOT EXISTS idx_sync_logs_status ON sync_logs (status); +CREATE INDEX IF NOT EXISTS idx_sync_logs_started_at ON sync_logs (started_at); + +-- Trigger function to auto-update updated_at column +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_data_sources_updated_at + BEFORE UPDATE ON data_sources + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER trg_sync_logs_updated_at + BEFORE UPDATE ON sync_logs + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); + +DO $$ BEGIN RAISE NOTICE '[Migration 000028] data_sources and sync_logs tables created successfully'; END $$;