diff --git a/.env.lite.example b/.env.lite.example index cfdcb56b..119a3d67 100644 --- a/.env.lite.example +++ b/.env.lite.example @@ -2,9 +2,18 @@ # 复制此文件为 .env.lite 并按需修改 # cp .env.lite.example .env.lite -GIN_MODE=release +GIN_MODE=debug + +# 日志级别，可选值：debug, info, warn, error, fatal +LOG_LEVEL=debug + +# 可选：日志文件绝对路径；未配置时，macOS .app 默认写入 +# ~/Library/Logs/WeKnora Lite/WeKnora Lite.log +# LOG_PATH=/absolute/path/to/weknora-lite.log # === 数据库 === +# macOS .app 中相对路径会自动重定向到 +# ~/Library/Application Support/WeKnora Lite/data/ DB_DRIVER=sqlite DB_PATH=./data/weknora.db diff --git a/.gitignore b/.gitignore index d778d957..c5241632 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ logs/ WeKnora WeKnora-lite +cmd/desktop/build/bin/ /models/ test/data/mswag.txt data/files/ diff --git a/cmd/desktop/build/appicon.png b/cmd/desktop/build/appicon.png new file mode 100644 index 00000000..63617fe4 Binary files /dev/null and b/cmd/desktop/build/appicon.png differ diff --git a/cmd/desktop/build/darwin/Info.plist b/cmd/desktop/build/darwin/Info.plist new file mode 100644 index 00000000..d17a7475 --- /dev/null +++ b/cmd/desktop/build/darwin/Info.plist @@ -0,0 +1,63 @@ + + + + CFBundlePackageType + APPL + CFBundleName + {{.Info.ProductName}} + CFBundleExecutable + {{.OutputFilename}} + CFBundleIdentifier + com.wails.{{.Name}} + CFBundleVersion + {{.Info.ProductVersion}} + CFBundleGetInfoString + {{.Info.Comments}} + CFBundleShortVersionString + {{.Info.ProductVersion}} + CFBundleIconFile + iconfile + LSMinimumSystemVersion + 10.13.0 + NSHighResolutionCapable + true + NSHumanReadableCopyright + {{.Info.Copyright}} + {{if .Info.FileAssociations}} + CFBundleDocumentTypes + + {{range .Info.FileAssociations}} + + CFBundleTypeExtensions + + {{.Ext}} + + CFBundleTypeName + {{.Name}} + CFBundleTypeRole + {{.Role}} + CFBundleTypeIconFile + {{.IconName}} + + {{end}} + + {{end}} + {{if .Info.Protocols}} + CFBundleURLTypes + + {{range .Info.Protocols}} + + CFBundleURLName + com.wails.{{.Scheme}} + CFBundleURLSchemes + + {{.Scheme}} + + CFBundleTypeRole + {{.Role}} + + {{end}} + + {{end}} + + diff --git a/cmd/desktop/main.go b/cmd/desktop/main.go index b529d525..95012e7a 100644 --- a/cmd/desktop/main.go +++ b/cmd/desktop/main.go @@ -33,6 +33,73 @@ import ( "golang.org/x/sys/unix" ) +// dragHandlerJS is injected into the webview on DomReady. +// It bypasses Wails' built-in CSS-variable-based drag detection (which uses +// getComputedStyle and has timing/inheritance issues with dynamic SPA content) +// and instead uses robust DOM-traversal via el.closest() plus a Y-position +// fallback for the macOS title-bar region on layout containers. The "drag" +// message is sent directly through the WKWebView script-message bridge, +// which the native Objective-C handler in WailsContext.m converts to +// [NSWindow performWindowDragWithEvent:]. +const dragHandlerJS = `(function(){ +if(window.__wkDragBound)return; +window.__wkDragBound=true; + +if(window.wails&&window.wails.flags){ + window.wails.flags.cssDragProperty='__disabled__'; + window.wails.flags.cssDragValue='__never__'; +} + +var TITLEBAR_H=38; + +var dragSel='.logo_row,.menu_top,.header,.header-title,.title-row,' + + '.dialogue-title,.section-header,.dialog-header,.sidebar-header,' + + '.document-header,.drag-region,[data-wails-drag]'; + +var noDragSel='button,a,input,select,textarea,[role="button"],' + + '.t-button,.t-input,.t-select,.t-textarea,' + + '.header-actions,.header-action-btn,.sidebar-toggle,.logo_box,' + + '.close-btn,.menu_item,.submenu,.submenu_item,.menu_bottom,' + + '.t-popup,.t-dropdown,.t-tooltip,.t-dialog,[data-no-drag]'; + +var layoutClasses=['main','chat','dialogue-wrap','kb-list-container', + 'agent-list-container','org-list-container','aside_box', + 'ks-container','settings-overlay','knowledge-layout', + 'faq-manager-wrapper','login-layout']; + +function sendDrag(){ + try{window.webkit.messageHandlers.external.postMessage('drag')} + catch(_){try{window.WailsInvoke('drag')}catch(e){}} +} + +function isLayoutEl(el){ + for(var i=0;i { - return noDragSelectors.some((selector) => el.closest(selector)); - }; - - const findDragTarget = (el) => { - for (const selector of dragSelectors) { - const match = el.closest(selector); - if (match) return match; - } - return null; - }; - - const applyDragMarkers = () => { - for (const selector of dragSelectors) { - document.querySelectorAll(selector).forEach((root) => { - if (!(root instanceof HTMLElement)) return; - root.style.setProperty('--wails-draggable', 'drag'); - root.querySelectorAll('*').forEach((child) => { - if (!(child instanceof HTMLElement)) return; - child.style.setProperty('--wails-draggable', 'drag'); - }); - }); - } - - for (const selector of noDragSelectors) { - document.querySelectorAll(selector).forEach((node) => { - if (!(node instanceof HTMLElement)) return; - node.style.setProperty('--wails-draggable', 'no-drag'); - node.querySelectorAll('*').forEach((child) => { - if (!(child instanceof HTMLElement)) return; - child.style.setProperty('--wails-draggable', 'no-drag'); - }); - }); - } - }; - - applyDragMarkers(); - const observer = new MutationObserver(() => { - applyDragMarkers(); - }); - observer.observe(document.body, { - childList: true, - subtree: true, - attributes: false - }); - - document.addEventListener('mousedown', (e) => { - if (e.button !== 0 || e.detail !== 1) return; - let target = e.target; - if (target instanceof Text) { - target = target.parentElement; - } - if (!(target instanceof Element)) return; - if (isNoDrag(target)) return; - const dragTarget = findDragTarget(target); - if (!dragTarget) return; - e.preventDefault(); - if (typeof window.WailsInvoke === 'function') { - window.WailsInvoke('drag'); - } - }, true); - })(); - ` - wailsruntime.WindowExecJS(ctx, css) + wailsruntime.WindowExecJS(ctx, dragHandlerJS) }, OnShutdown: app.shutdown, Bind: []interface{}{ @@ -345,6 +282,83 @@ func (a *App) shutdown(ctx context.Context) { a.shutdownCh <- struct{}{} } +func configureDesktopStorage(execPath string) { + if execPath == "" || !strings.Contains(execPath, ".app/Contents/MacOS") { + return + } + + appSupportDir, err := defaultMacAppSupportDir(execPath) + if err != nil { + logger.Warnf(context.Background(), "Failed to resolve app support dir: %v", err) + return + } + + legacyResourcesDir := filepath.Join(filepath.Dir(filepath.Dir(execPath)), "Resources") + targetDataDir := filepath.Join(appSupportDir, "data") + migrateLegacyDesktopData(legacyResourcesDir, targetDataDir) + + dbPath := resolveDesktopDataPath(os.Getenv("DB_PATH"), filepath.Join("data", "weknora.db"), appSupportDir) + filesPath := resolveDesktopDataPath(os.Getenv("LOCAL_STORAGE_BASE_DIR"), filepath.Join("data", "files"), appSupportDir) + + if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil { + logger.Warnf(context.Background(), "Failed to create desktop DB directory %s: %v", filepath.Dir(dbPath), err) + } + if err := os.MkdirAll(filesPath, 0o755); err != nil { + logger.Warnf(context.Background(), "Failed to create desktop files directory %s: %v", filesPath, err) + } + + _ = os.Setenv("DB_PATH", dbPath) + _ = os.Setenv("LOCAL_STORAGE_BASE_DIR", filesPath) +} + +func defaultMacAppSupportDir(execPath string) (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + + appName := "WeKnora Lite" + if idx := strings.Index(execPath, ".app/Contents/MacOS"); idx >= 0 { + bundleName := filepath.Base(execPath[:idx+4]) + if trimmed := strings.TrimSuffix(bundleName, ".app"); trimmed != "" { + appName = trimmed + } + } + + return filepath.Join(homeDir, "Library", "Application Support", appName), nil +} + +func resolveDesktopDataPath(rawPath, defaultRelativePath, appSupportDir string) string { + trimmed := strings.TrimSpace(rawPath) + if trimmed == "" { + trimmed = defaultRelativePath + } + if filepath.IsAbs(trimmed) { + return filepath.Clean(trimmed) + } + trimmed = strings.TrimPrefix(trimmed, "."+string(filepath.Separator)) + return filepath.Join(appSupportDir, filepath.Clean(trimmed)) +} + +func migrateLegacyDesktopData(resourcesDir, targetDataDir string) { + legacyDataDir := filepath.Join(resourcesDir, "data") + if info, err := os.Stat(legacyDataDir); err != nil || !info.IsDir() { + return + } + if _, err := os.Stat(targetDataDir); err == nil { + return + } + if err := os.MkdirAll(filepath.Dir(targetDataDir), 0o755); err != nil { + logger.Warnf(context.Background(), "Failed to create app support parent dir %s: %v", filepath.Dir(targetDataDir), err) + return + } + if err := os.Rename(legacyDataDir, targetDataDir); err != nil { + logger.Warnf(context.Background(), "Failed to migrate legacy desktop data from %s to %s: %v", legacyDataDir, targetDataDir, err) + return + } + logger.Infof(context.Background(), "Migrated legacy desktop data to %s", targetDataDir) +} + func listenWithRetry(addr string, maxRetries int, baseDelay time.Duration) (net.Listener, error) { lc := net.ListenConfig{ Control: func(network, address string, c syscall.RawConn) error { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7b782e78..7b64e27e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "knowledage-base", - "version": "0.3.4", + "version": "0.3.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "knowledage-base", - "version": "0.3.4", + "version": "0.3.6", "dependencies": { "@microsoft/fetch-event-source": "^2.0.1", "@types/dompurify": "^3.0.5", diff --git a/frontend/package.json.md5 b/frontend/package.json.md5 new file mode 100755 index 00000000..e994af2d --- /dev/null +++ b/frontend/package.json.md5 @@ -0,0 +1 @@ +a8c281c741c2499a02564295ccfb3f91 \ No newline at end of file diff --git a/frontend/src/api/chat/streame.ts b/frontend/src/api/chat/streame.ts index ec789d92..e1963b07 100644 --- a/frontend/src/api/chat/streame.ts +++ b/frontend/src/api/chat/streame.ts @@ -2,6 +2,7 @@ import { fetchEventSource } from '@microsoft/fetch-event-source' import { ref, type Ref, onUnmounted, nextTick } from 'vue' import { generateRandomString } from '@/utils/index'; import i18n from '@/i18n'; +import { getApiBaseUrl } from '@/utils/api-base'; @@ -37,7 +38,7 @@ export function useStream() { isLoading.value = true; // 获取API配置 - const apiUrl = import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080"; + const apiUrl = getApiBaseUrl(); // 获取JWT Token const token = localStorage.getItem('weknora_token'); diff --git a/frontend/src/components/IMChannelPanel.vue b/frontend/src/components/IMChannelPanel.vue index 787d7e9e..ea4e0597 100644 --- a/frontend/src/components/IMChannelPanel.vue +++ b/frontend/src/components/IMChannelPanel.vue @@ -419,7 +419,7 @@ async function loadChannels() { } function getCallbackUrl(channel: IMChannel): string { - const base = import.meta.env.VITE_IS_DOCKER ? window.location.origin : 'http://127.0.0.1:8080'; + const base = window.location.origin; return `${base}/api/v1/im/callback/${channel.id}`; } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 5f99a453..82034586 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -3458,11 +3458,15 @@ export default { untitled: 'Untitled', resourceLoadFailed: 'Failed to load resources', noResources: 'No wiki spaces found', - noResourcesDesc: 'The app needs wiki access via a group chat to fetch content', + noResourcesDesc: 'The app needs wiki access via a group chat to fetch content', + noResourcesDesc_notion: 'The app needs Notion page access permissions to fetch content', retryLoadResources: 'Retry', guideStep1: 'Create a group chat in Feishu, then add your app as a bot in the group settings', guideStep2: 'Open wiki "Settings" > "Member Settings" > "Add Member", search for the group chat and add it', guideStep3: 'Ensure the group chat role is at least "Can Read", then come back and click Retry', + guideStep1_notion: 'Open the page or database you want to sync in Notion', + guideStep2_notion: 'Click the "···" menu at the top right, select "Connect to" or "Add connections"', + guideStep3_notion: 'Search and select your Integration app, then come back and click Retry', permissionDocLink: 'View Feishu wiki permission docs', syncScheduleLabel: 'Sync schedule', conflictLabel: 'Conflict strategy', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 596c7222..80d14db4 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -3510,11 +3510,15 @@ export default { untitled: "제목 없음", resourceLoadFailed: "리소스 목록 로드 실패", noResources: "동기화 가능한 위키 공간을 찾을 수 없습니다", - noResourcesHint: "먼저 데이터 소스를 저장하고 권한 설정을 확인하세요", - permissionHint: "앱이 위키 공간의 멤버로 추가되어야 콘텐츠에 접근할 수 있습니다:", - permissionStep1: "페이슈를 열고 동기화할 위키 공간으로 이동", - permissionStep2: "오른쪽 상단 '...' > '설정' > '멤버 관리' 클릭", - permissionStep3: "페이슈 앱을 검색하여 위키 공간 멤버로 추가", + noResourcesDesc: "앱이 콘텐츠를 가져오려면 그룹 채팅을 통해 위키 접근 권한을 얻어야 합니다", + noResourcesDesc_notion: "앱이 콘텐츠를 가져오려면 Notion 페이지 접근 권한이 필요합니다", + retryLoadResources: "다시 시도", + guideStep1: "Feishu에서 그룹 채팅을 만들고 그룹 설정의 '그룹 봇'에 앱을 추가하세요", + guideStep2: "위키 '설정' > '멤버 설정' > '멤버 추가'를 열고 해당 그룹 채팅을 검색하여 추가하세요", + guideStep3: "그룹 채팅 역할이 최소 '읽기 가능'인지 확인한 후, 돌아와서 다시 시도를 클릭하세요", + guideStep1_notion: "Notion에서 동기화하려는 페이지나 데이터베이스를 엽니다", + guideStep2_notion: "오른쪽 상단의 '···' 메뉴를 클릭하고 'Connect to' 또는 'Add connections'를 선택합니다", + guideStep3_notion: "Integration 앱을 검색하여 선택한 후, 돌아와서 다시 시도를 클릭하세요", permissionDocLink: "페이슈 위키 권한 설정 문서 보기", syncScheduleLabel: "동기화 주기", conflictLabel: "충돌 전략", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 9410e14f..94908497 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -3417,11 +3417,15 @@ export default { untitled: 'Без названия', resourceLoadFailed: 'Не удалось загрузить список ресурсов', noResources: 'Пространства вики не найдены', - noResourcesHint: 'Сначала сохраните источник данных и проверьте права доступа', - permissionHint: 'Приложение должно быть добавлено как участник пространства вики для доступа к контенту:', - permissionStep1: 'Откройте Feishu и перейдите в пространство вики для синхронизации', - permissionStep2: 'Нажмите "..." > "Настройки" > "Управление участниками"', - permissionStep3: 'Найдите и добавьте приложение Feishu как участника пространства', + noResourcesDesc: 'Приложению требуется доступ к вики через групповой чат для получения контента', + noResourcesDesc_notion: 'Приложению требуются права доступа к странице Notion для получения контента', + retryLoadResources: 'Повторить', + guideStep1: 'Создайте групповой чат в Feishu, затем добавьте ваше приложение как бота в настройках группы', + guideStep2: 'Откройте вики "Настройки" > "Управление участниками" > "Добавить участника", найдите групповой чат и добавьте его', + guideStep3: 'Убедитесь, что роль группового чата как минимум "Может читать", затем вернитесь и нажмите Повторить', + guideStep1_notion: 'Откройте страницу или базу данных, которую хотите синхронизировать в Notion', + guideStep2_notion: 'Нажмите меню «···» в правом верхнем углу, выберите «Connect to» или «Add connections»', + guideStep3_notion: 'Найдите и выберите ваше интеграционное приложение, затем вернитесь и нажмите Повторить', permissionDocLink: 'Документация по настройке прав доступа', syncScheduleLabel: 'Расписание синхронизации', conflictLabel: 'Стратегия конфликтов', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 08130813..d6e6b813 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -3457,11 +3457,15 @@ export default { untitled: "无标题", resourceLoadFailed: "加载资源列表失败", noResources: "未找到可同步的知识库空间", - noResourcesDesc: "应用需要通过群聊获得知识库访问权限才能拉取内容", + noResourcesDesc: "应用需要通过群聊获得知识库访问权限才能拉取内容", + noResourcesDesc_notion: "应用需要获得 Notion 页面的访问权限才能拉取内容", retryLoadResources: "重新加载", guideStep1: "在飞书中创建一个群聊，在群设置「群机器人」中添加你的应用", guideStep2: "打开知识库「设置」→「成员设置」→ 添加成员，搜索该群聊名称并添加", guideStep3: "确保群聊角色至少为「可阅读」，然后回到这里点重新加载", + guideStep1_notion: "在 Notion 中打开你想要同步的页面或数据库", + guideStep2_notion: "点击右上角的「···」菜单，选择「Connect to」或「Add connections」", + guideStep3_notion: "搜索并选择你的集成应用（Integration），然后回到这里点重新加载", permissionDocLink: "查看飞书知识库权限配置文档", syncScheduleLabel: "同步频率", conflictLabel: "冲突策略", diff --git a/frontend/src/stores/settings.ts b/frontend/src/stores/settings.ts index 3aaed235..010a49e4 100644 --- a/frontend/src/stores/settings.ts +++ b/frontend/src/stores/settings.ts @@ -1,5 +1,6 @@ import { defineStore } from "pinia"; import { BUILTIN_QUICK_ANSWER_ID, BUILTIN_SMART_REASONING_ID } from "@/api/agent"; +import { getApiBaseUrl } from "@/utils/api-base"; // 定义设置接口 interface Settings { @@ -63,7 +64,7 @@ interface OllamaConfig { // 默认设置 const defaultSettings: Settings = { - endpoint: import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080", + endpoint: getApiBaseUrl(), apiKey: "", knowledgeBaseId: "", isAgentEnabled: false, diff --git a/frontend/src/utils/api-base.ts b/frontend/src/utils/api-base.ts new file mode 100644 index 00000000..5e3908c8 --- /dev/null +++ b/frontend/src/utils/api-base.ts @@ -0,0 +1,5 @@ +export function getApiBaseUrl(): string { + // Use same-origin requests by default. + // In local Vite dev, `vite.config.ts` proxies `/api` to the Go backend. + return ''; +} diff --git a/frontend/src/utils/request.ts b/frontend/src/utils/request.ts index e5df898d..b6d77cf8 100644 --- a/frontend/src/utils/request.ts +++ b/frontend/src/utils/request.ts @@ -2,11 +2,12 @@ import axios from "axios"; import { generateRandomString } from "./index"; import i18n from '@/i18n' +import { getApiBaseUrl } from './api-base'; const t = (key: string) => i18n.global.t(key) // API基础URL -const BASE_URL = import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080"; +const BASE_URL = getApiBaseUrl(); // 创建Axios实例 diff --git a/frontend/src/views/chat/components/AgentStreamDisplay.vue b/frontend/src/views/chat/components/AgentStreamDisplay.vue index 8fbbc3c1..2e920524 100644 --- a/frontend/src/views/chat/components/AgentStreamDisplay.vue +++ b/frontend/src/views/chat/components/AgentStreamDisplay.vue @@ -1000,9 +1000,15 @@ const handleCitationActivate = (el: HTMLElement) => { const url = el.getAttribute('data-url'); if (!url) return; try { - const newWindow = window.open(url, '_blank', 'noopener,noreferrer'); - if (!newWindow) { - window.location.assign(url); + // @ts-ignore: Wails runtime check + if (window.runtime && window.runtime.BrowserOpenURL) { + // @ts-ignore + window.runtime.BrowserOpenURL(url); + } else { + const newWindow = window.open(url, '_blank', 'noopener,noreferrer'); + if (!newWindow) { + window.location.assign(url); + } } } catch { window.location.assign(url); @@ -1200,10 +1206,8 @@ const onRootClick = (e: Event) => { // Handle web citation clicks const webEl = target.closest?.('.citation-web') as HTMLElement | null; if (webEl && webEl.getAttribute('data-url')) { - if (!(webEl instanceof HTMLAnchorElement)) { - e.preventDefault(); - handleCitationActivate(webEl); - } + e.preventDefault(); + handleCitationActivate(webEl); return; } @@ -1223,6 +1227,18 @@ const onRootClick = (e: Event) => { } return; } + + // Handle generic a clicks (especially in Wails desktop) + const aEl = target.closest?.('a') as HTMLAnchorElement | null; + // @ts-ignore + if (aEl && aEl.href && window.runtime && window.runtime.BrowserOpenURL) { + if (aEl.href.startsWith('http://') || aEl.href.startsWith('https://')) { + e.preventDefault(); + // @ts-ignore + window.runtime.BrowserOpenURL(aEl.href); + return; + } + } }; const onRootKeydown = (e: KeyboardEvent) => { @@ -1233,15 +1249,8 @@ const onRootKeydown = (e: KeyboardEvent) => { const webEl = target.closest?.('.citation-web') as HTMLElement | null; if (webEl) { if (e.key === 'Enter' || e.key === ' ') { - if (webEl instanceof HTMLAnchorElement && e.key === 'Enter') { - return; - } e.preventDefault(); - if (webEl instanceof HTMLAnchorElement) { - webEl.click(); - } else { - handleCitationActivate(webEl); - } + handleCitationActivate(webEl); } return; } diff --git a/frontend/src/wailsjs/runtime/package.json b/frontend/src/wailsjs/runtime/package.json new file mode 100644 index 00000000..1e7c8a5d --- /dev/null +++ b/frontend/src/wailsjs/runtime/package.json @@ -0,0 +1,24 @@ +{ + "name": "@wailsapp/runtime", + "version": "2.0.0", + "description": "Wails Javascript runtime library", + "main": "runtime.js", + "types": "runtime.d.ts", + "scripts": { + }, + "repository": { + "type": "git", + "url": "git+https://github.com/wailsapp/wails.git" + }, + "keywords": [ + "Wails", + "Javascript", + "Go" + ], + "author": "Lea Anthony ", + "license": "MIT", + "bugs": { + "url": "https://github.com/wailsapp/wails/issues" + }, + "homepage": "https://github.com/wailsapp/wails#readme" +} diff --git a/frontend/src/wailsjs/runtime/runtime.d.ts b/frontend/src/wailsjs/runtime/runtime.d.ts new file mode 100644 index 00000000..3bbea848 --- /dev/null +++ b/frontend/src/wailsjs/runtime/runtime.d.ts @@ -0,0 +1,330 @@ +/* + _ __ _ __ +| | / /___ _(_) /____ +| | /| / / __ `/ / / ___/ +| |/ |/ / /_/ / / (__ ) +|__/|__/\__,_/_/_/____/ +The electron alternative for Go +(c) Lea Anthony 2019-present +*/ + +export interface Position { + x: number; + y: number; +} + +export interface Size { + w: number; + h: number; +} + +export interface Screen { + isCurrent: boolean; + isPrimary: boolean; + width : number + height : number +} + +// Environment information such as platform, buildtype, ... +export interface EnvironmentInfo { + buildType: string; + platform: string; + arch: string; +} + +// [EventsEmit](https://wails.io/docs/reference/runtime/events#eventsemit) +// emits the given event. Optional data may be passed with the event. +// This will trigger any event listeners. +export function EventsEmit(eventName: string, ...data: any): void; + +// [EventsOn](https://wails.io/docs/reference/runtime/events#eventson) sets up a listener for the given event name. +export function EventsOn(eventName: string, callback: (...data: any) => void): () => void; + +// [EventsOnMultiple](https://wails.io/docs/reference/runtime/events#eventsonmultiple) +// sets up a listener for the given event name, but will only trigger a given number times. +export function EventsOnMultiple(eventName: string, callback: (...data: any) => void, maxCallbacks: number): () => void; + +// [EventsOnce](https://wails.io/docs/reference/runtime/events#eventsonce) +// sets up a listener for the given event name, but will only trigger once. +export function EventsOnce(eventName: string, callback: (...data: any) => void): () => void; + +// [EventsOff](https://wails.io/docs/reference/runtime/events#eventsoff) +// unregisters the listener for the given event name. +export function EventsOff(eventName: string, ...additionalEventNames: string[]): void; + +// [EventsOffAll](https://wails.io/docs/reference/runtime/events#eventsoffall) +// unregisters all listeners. +export function EventsOffAll(): void; + +// [LogPrint](https://wails.io/docs/reference/runtime/log#logprint) +// logs the given message as a raw message +export function LogPrint(message: string): void; + +// [LogTrace](https://wails.io/docs/reference/runtime/log#logtrace) +// logs the given message at the `trace` log level. +export function LogTrace(message: string): void; + +// [LogDebug](https://wails.io/docs/reference/runtime/log#logdebug) +// logs the given message at the `debug` log level. +export function LogDebug(message: string): void; + +// [LogError](https://wails.io/docs/reference/runtime/log#logerror) +// logs the given message at the `error` log level. +export function LogError(message: string): void; + +// [LogFatal](https://wails.io/docs/reference/runtime/log#logfatal) +// logs the given message at the `fatal` log level. +// The application will quit after calling this method. +export function LogFatal(message: string): void; + +// [LogInfo](https://wails.io/docs/reference/runtime/log#loginfo) +// logs the given message at the `info` log level. +export function LogInfo(message: string): void; + +// [LogWarning](https://wails.io/docs/reference/runtime/log#logwarning) +// logs the given message at the `warning` log level. +export function LogWarning(message: string): void; + +// [WindowReload](https://wails.io/docs/reference/runtime/window#windowreload) +// Forces a reload by the main application as well as connected browsers. +export function WindowReload(): void; + +// [WindowReloadApp](https://wails.io/docs/reference/runtime/window#windowreloadapp) +// Reloads the application frontend. +export function WindowReloadApp(): void; + +// [WindowSetAlwaysOnTop](https://wails.io/docs/reference/runtime/window#windowsetalwaysontop) +// Sets the window AlwaysOnTop or not on top. +export function WindowSetAlwaysOnTop(b: boolean): void; + +// [WindowSetSystemDefaultTheme](https://wails.io/docs/next/reference/runtime/window#windowsetsystemdefaulttheme) +// *Windows only* +// Sets window theme to system default (dark/light). +export function WindowSetSystemDefaultTheme(): void; + +// [WindowSetLightTheme](https://wails.io/docs/next/reference/runtime/window#windowsetlighttheme) +// *Windows only* +// Sets window to light theme. +export function WindowSetLightTheme(): void; + +// [WindowSetDarkTheme](https://wails.io/docs/next/reference/runtime/window#windowsetdarktheme) +// *Windows only* +// Sets window to dark theme. +export function WindowSetDarkTheme(): void; + +// [WindowCenter](https://wails.io/docs/reference/runtime/window#windowcenter) +// Centers the window on the monitor the window is currently on. +export function WindowCenter(): void; + +// [WindowSetTitle](https://wails.io/docs/reference/runtime/window#windowsettitle) +// Sets the text in the window title bar. +export function WindowSetTitle(title: string): void; + +// [WindowFullscreen](https://wails.io/docs/reference/runtime/window#windowfullscreen) +// Makes the window full screen. +export function WindowFullscreen(): void; + +// [WindowUnfullscreen](https://wails.io/docs/reference/runtime/window#windowunfullscreen) +// Restores the previous window dimensions and position prior to full screen. +export function WindowUnfullscreen(): void; + +// [WindowIsFullscreen](https://wails.io/docs/reference/runtime/window#windowisfullscreen) +// Returns the state of the window, i.e. whether the window is in full screen mode or not. +export function WindowIsFullscreen(): Promise; + +// [WindowSetSize](https://wails.io/docs/reference/runtime/window#windowsetsize) +// Sets the width and height of the window. +export function WindowSetSize(width: number, height: number): void; + +// [WindowGetSize](https://wails.io/docs/reference/runtime/window#windowgetsize) +// Gets the width and height of the window. +export function WindowGetSize(): Promise; + +// [WindowSetMaxSize](https://wails.io/docs/reference/runtime/window#windowsetmaxsize) +// Sets the maximum window size. Will resize the window if the window is currently larger than the given dimensions. +// Setting a size of 0,0 will disable this constraint. +export function WindowSetMaxSize(width: number, height: number): void; + +// [WindowSetMinSize](https://wails.io/docs/reference/runtime/window#windowsetminsize) +// Sets the minimum window size. Will resize the window if the window is currently smaller than the given dimensions. +// Setting a size of 0,0 will disable this constraint. +export function WindowSetMinSize(width: number, height: number): void; + +// [WindowSetPosition](https://wails.io/docs/reference/runtime/window#windowsetposition) +// Sets the window position relative to the monitor the window is currently on. +export function WindowSetPosition(x: number, y: number): void; + +// [WindowGetPosition](https://wails.io/docs/reference/runtime/window#windowgetposition) +// Gets the window position relative to the monitor the window is currently on. +export function WindowGetPosition(): Promise; + +// [WindowHide](https://wails.io/docs/reference/runtime/window#windowhide) +// Hides the window. +export function WindowHide(): void; + +// [WindowShow](https://wails.io/docs/reference/runtime/window#windowshow) +// Shows the window, if it is currently hidden. +export function WindowShow(): void; + +// [WindowMaximise](https://wails.io/docs/reference/runtime/window#windowmaximise) +// Maximises the window to fill the screen. +export function WindowMaximise(): void; + +// [WindowToggleMaximise](https://wails.io/docs/reference/runtime/window#windowtogglemaximise) +// Toggles between Maximised and UnMaximised. +export function WindowToggleMaximise(): void; + +// [WindowUnmaximise](https://wails.io/docs/reference/runtime/window#windowunmaximise) +// Restores the window to the dimensions and position prior to maximising. +export function WindowUnmaximise(): void; + +// [WindowIsMaximised](https://wails.io/docs/reference/runtime/window#windowismaximised) +// Returns the state of the window, i.e. whether the window is maximised or not. +export function WindowIsMaximised(): Promise; + +// [WindowMinimise](https://wails.io/docs/reference/runtime/window#windowminimise) +// Minimises the window. +export function WindowMinimise(): void; + +// [WindowUnminimise](https://wails.io/docs/reference/runtime/window#windowunminimise) +// Restores the window to the dimensions and position prior to minimising. +export function WindowUnminimise(): void; + +// [WindowIsMinimised](https://wails.io/docs/reference/runtime/window#windowisminimised) +// Returns the state of the window, i.e. whether the window is minimised or not. +export function WindowIsMinimised(): Promise; + +// [WindowIsNormal](https://wails.io/docs/reference/runtime/window#windowisnormal) +// Returns the state of the window, i.e. whether the window is normal or not. +export function WindowIsNormal(): Promise; + +// [WindowSetBackgroundColour](https://wails.io/docs/reference/runtime/window#windowsetbackgroundcolour) +// Sets the background colour of the window to the given RGBA colour definition. This colour will show through for all transparent pixels. +export function WindowSetBackgroundColour(R: number, G: number, B: number, A: number): void; + +// [ScreenGetAll](https://wails.io/docs/reference/runtime/window#screengetall) +// Gets the all screens. Call this anew each time you want to refresh data from the underlying windowing system. +export function ScreenGetAll(): Promise; + +// [BrowserOpenURL](https://wails.io/docs/reference/runtime/browser#browseropenurl) +// Opens the given URL in the system browser. +export function BrowserOpenURL(url: string): void; + +// [Environment](https://wails.io/docs/reference/runtime/intro#environment) +// Returns information about the environment +export function Environment(): Promise; + +// [Quit](https://wails.io/docs/reference/runtime/intro#quit) +// Quits the application. +export function Quit(): void; + +// [Hide](https://wails.io/docs/reference/runtime/intro#hide) +// Hides the application. +export function Hide(): void; + +// [Show](https://wails.io/docs/reference/runtime/intro#show) +// Shows the application. +export function Show(): void; + +// [ClipboardGetText](https://wails.io/docs/reference/runtime/clipboard#clipboardgettext) +// Returns the current text stored on clipboard +export function ClipboardGetText(): Promise; + +// [ClipboardSetText](https://wails.io/docs/reference/runtime/clipboard#clipboardsettext) +// Sets a text on the clipboard +export function ClipboardSetText(text: string): Promise; + +// [OnFileDrop](https://wails.io/docs/reference/runtime/draganddrop#onfiledrop) +// OnFileDrop listens to drag and drop events and calls the callback with the coordinates of the drop and an array of path strings. +export function OnFileDrop(callback: (x: number, y: number ,paths: string[]) => void, useDropTarget: boolean) :void + +// [OnFileDropOff](https://wails.io/docs/reference/runtime/draganddrop#dragandddropoff) +// OnFileDropOff removes the drag and drop listeners and handlers. +export function OnFileDropOff() :void + +// Check if the file path resolver is available +export function CanResolveFilePaths(): boolean; + +// Resolves file paths for an array of files +export function ResolveFilePaths(files: File[]): void + +// Notification types +export interface NotificationOptions { + id: string; + title: string; + subtitle?: string; // macOS and Linux only + body?: string; + categoryId?: string; + data?: { [key: string]: any }; +} + +export interface NotificationAction { + id?: string; + title?: string; + destructive?: boolean; // macOS-specific +} + +export interface NotificationCategory { + id?: string; + actions?: NotificationAction[]; + hasReplyField?: boolean; + replyPlaceholder?: string; + replyButtonTitle?: string; +} + +// [InitializeNotifications](https://wails.io/docs/reference/runtime/notification#initializenotifications) +// Initializes the notification service for the application. +// This must be called before sending any notifications. +export function InitializeNotifications(): Promise; + +// [CleanupNotifications](https://wails.io/docs/reference/runtime/notification#cleanupnotifications) +// Cleans up notification resources and releases any held connections. +export function CleanupNotifications(): Promise; + +// [IsNotificationAvailable](https://wails.io/docs/reference/runtime/notification#isnotificationavailable) +// Checks if notifications are available on the current platform. +export function IsNotificationAvailable(): Promise; + +// [RequestNotificationAuthorization](https://wails.io/docs/reference/runtime/notification#requestnotificationauthorization) +// Requests notification authorization from the user (macOS only). +export function RequestNotificationAuthorization(): Promise; + +// [CheckNotificationAuthorization](https://wails.io/docs/reference/runtime/notification#checknotificationauthorization) +// Checks the current notification authorization status (macOS only). +export function CheckNotificationAuthorization(): Promise; + +// [SendNotification](https://wails.io/docs/reference/runtime/notification#sendnotification) +// Sends a basic notification with the given options. +export function SendNotification(options: NotificationOptions): Promise; + +// [SendNotificationWithActions](https://wails.io/docs/reference/runtime/notification#sendnotificationwithactions) +// Sends a notification with action buttons. Requires a registered category. +export function SendNotificationWithActions(options: NotificationOptions): Promise; + +// [RegisterNotificationCategory](https://wails.io/docs/reference/runtime/notification#registernotificationcategory) +// Registers a notification category that can be used with SendNotificationWithActions. +export function RegisterNotificationCategory(category: NotificationCategory): Promise; + +// [RemoveNotificationCategory](https://wails.io/docs/reference/runtime/notification#removenotificationcategory) +// Removes a previously registered notification category. +export function RemoveNotificationCategory(categoryId: string): Promise; + +// [RemoveAllPendingNotifications](https://wails.io/docs/reference/runtime/notification#removeallpendingnotifications) +// Removes all pending notifications from the notification center. +export function RemoveAllPendingNotifications(): Promise; + +// [RemovePendingNotification](https://wails.io/docs/reference/runtime/notification#removependingnotification) +// Removes a specific pending notification by its identifier. +export function RemovePendingNotification(identifier: string): Promise; + +// [RemoveAllDeliveredNotifications](https://wails.io/docs/reference/runtime/notification#removealldeliverednotifications) +// Removes all delivered notifications from the notification center. +export function RemoveAllDeliveredNotifications(): Promise; + +// [RemoveDeliveredNotification](https://wails.io/docs/reference/runtime/notification#removedeliverednotification) +// Removes a specific delivered notification by its identifier. +export function RemoveDeliveredNotification(identifier: string): Promise; + +// [RemoveNotification](https://wails.io/docs/reference/runtime/notification#removenotification) +// Removes a notification by its identifier (cross-platform convenience function). +export function RemoveNotification(identifier: string): Promise; \ No newline at end of file diff --git a/frontend/src/wailsjs/runtime/runtime.js b/frontend/src/wailsjs/runtime/runtime.js new file mode 100644 index 00000000..556621ee --- /dev/null +++ b/frontend/src/wailsjs/runtime/runtime.js @@ -0,0 +1,298 @@ +/* + _ __ _ __ +| | / /___ _(_) /____ +| | /| / / __ `/ / / ___/ +| |/ |/ / /_/ / / (__ ) +|__/|__/\__,_/_/_/____/ +The electron alternative for Go +(c) Lea Anthony 2019-present +*/ + +export function LogPrint(message) { + window.runtime.LogPrint(message); +} + +export function LogTrace(message) { + window.runtime.LogTrace(message); +} + +export function LogDebug(message) { + window.runtime.LogDebug(message); +} + +export function LogInfo(message) { + window.runtime.LogInfo(message); +} + +export function LogWarning(message) { + window.runtime.LogWarning(message); +} + +export function LogError(message) { + window.runtime.LogError(message); +} + +export function LogFatal(message) { + window.runtime.LogFatal(message); +} + +export function EventsOnMultiple(eventName, callback, maxCallbacks) { + return window.runtime.EventsOnMultiple(eventName, callback, maxCallbacks); +} + +export function EventsOn(eventName, callback) { + return EventsOnMultiple(eventName, callback, -1); +} + +export function EventsOff(eventName, ...additionalEventNames) { + return window.runtime.EventsOff(eventName, ...additionalEventNames); +} + +export function EventsOffAll() { + return window.runtime.EventsOffAll(); +} + +export function EventsOnce(eventName, callback) { + return EventsOnMultiple(eventName, callback, 1); +} + +export function EventsEmit(eventName) { + let args = [eventName].slice.call(arguments); + return window.runtime.EventsEmit.apply(null, args); +} + +export function WindowReload() { + window.runtime.WindowReload(); +} + +export function WindowReloadApp() { + window.runtime.WindowReloadApp(); +} + +export function WindowSetAlwaysOnTop(b) { + window.runtime.WindowSetAlwaysOnTop(b); +} + +export function WindowSetSystemDefaultTheme() { + window.runtime.WindowSetSystemDefaultTheme(); +} + +export function WindowSetLightTheme() { + window.runtime.WindowSetLightTheme(); +} + +export function WindowSetDarkTheme() { + window.runtime.WindowSetDarkTheme(); +} + +export function WindowCenter() { + window.runtime.WindowCenter(); +} + +export function WindowSetTitle(title) { + window.runtime.WindowSetTitle(title); +} + +export function WindowFullscreen() { + window.runtime.WindowFullscreen(); +} + +export function WindowUnfullscreen() { + window.runtime.WindowUnfullscreen(); +} + +export function WindowIsFullscreen() { + return window.runtime.WindowIsFullscreen(); +} + +export function WindowGetSize() { + return window.runtime.WindowGetSize(); +} + +export function WindowSetSize(width, height) { + window.runtime.WindowSetSize(width, height); +} + +export function WindowSetMaxSize(width, height) { + window.runtime.WindowSetMaxSize(width, height); +} + +export function WindowSetMinSize(width, height) { + window.runtime.WindowSetMinSize(width, height); +} + +export function WindowSetPosition(x, y) { + window.runtime.WindowSetPosition(x, y); +} + +export function WindowGetPosition() { + return window.runtime.WindowGetPosition(); +} + +export function WindowHide() { + window.runtime.WindowHide(); +} + +export function WindowShow() { + window.runtime.WindowShow(); +} + +export function WindowMaximise() { + window.runtime.WindowMaximise(); +} + +export function WindowToggleMaximise() { + window.runtime.WindowToggleMaximise(); +} + +export function WindowUnmaximise() { + window.runtime.WindowUnmaximise(); +} + +export function WindowIsMaximised() { + return window.runtime.WindowIsMaximised(); +} + +export function WindowMinimise() { + window.runtime.WindowMinimise(); +} + +export function WindowUnminimise() { + window.runtime.WindowUnminimise(); +} + +export function WindowSetBackgroundColour(R, G, B, A) { + window.runtime.WindowSetBackgroundColour(R, G, B, A); +} + +export function ScreenGetAll() { + return window.runtime.ScreenGetAll(); +} + +export function WindowIsMinimised() { + return window.runtime.WindowIsMinimised(); +} + +export function WindowIsNormal() { + return window.runtime.WindowIsNormal(); +} + +export function BrowserOpenURL(url) { + window.runtime.BrowserOpenURL(url); +} + +export function Environment() { + return window.runtime.Environment(); +} + +export function Quit() { + window.runtime.Quit(); +} + +export function Hide() { + window.runtime.Hide(); +} + +export function Show() { + window.runtime.Show(); +} + +export function ClipboardGetText() { + return window.runtime.ClipboardGetText(); +} + +export function ClipboardSetText(text) { + return window.runtime.ClipboardSetText(text); +} + +/** + * Callback for OnFileDrop returns a slice of file path strings when a drop is finished. + * + * @export + * @callback OnFileDropCallback + * @param {number} x - x coordinate of the drop + * @param {number} y - y coordinate of the drop + * @param {string[]} paths - A list of file paths. + */ + +/** + * OnFileDrop listens to drag and drop events and calls the callback with the coordinates of the drop and an array of path strings. + * + * @export + * @param {OnFileDropCallback} callback - Callback for OnFileDrop returns a slice of file path strings when a drop is finished. + * @param {boolean} [useDropTarget=true] - Only call the callback when the drop finished on an element that has the drop target style. (--wails-drop-target) + */ +export function OnFileDrop(callback, useDropTarget) { + return window.runtime.OnFileDrop(callback, useDropTarget); +} + +/** + * OnFileDropOff removes the drag and drop listeners and handlers. + */ +export function OnFileDropOff() { + return window.runtime.OnFileDropOff(); +} + +export function CanResolveFilePaths() { + return window.runtime.CanResolveFilePaths(); +} + +export function ResolveFilePaths(files) { + return window.runtime.ResolveFilePaths(files); +} + +export function InitializeNotifications() { + return window.runtime.InitializeNotifications(); +} + +export function CleanupNotifications() { + return window.runtime.CleanupNotifications(); +} + +export function IsNotificationAvailable() { + return window.runtime.IsNotificationAvailable(); +} + +export function RequestNotificationAuthorization() { + return window.runtime.RequestNotificationAuthorization(); +} + +export function CheckNotificationAuthorization() { + return window.runtime.CheckNotificationAuthorization(); +} + +export function SendNotification(options) { + return window.runtime.SendNotification(options); +} + +export function SendNotificationWithActions(options) { + return window.runtime.SendNotificationWithActions(options); +} + +export function RegisterNotificationCategory(category) { + return window.runtime.RegisterNotificationCategory(category); +} + +export function RemoveNotificationCategory(categoryId) { + return window.runtime.RemoveNotificationCategory(categoryId); +} + +export function RemoveAllPendingNotifications() { + return window.runtime.RemoveAllPendingNotifications(); +} + +export function RemovePendingNotification(identifier) { + return window.runtime.RemovePendingNotification(identifier); +} + +export function RemoveAllDeliveredNotifications() { + return window.runtime.RemoveAllDeliveredNotifications(); +} + +export function RemoveDeliveredNotification(identifier) { + return window.runtime.RemoveDeliveredNotification(identifier); +} + +export function RemoveNotification(identifier) { + return window.runtime.RemoveNotification(identifier); +} \ No newline at end of file diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.go index 88e41126..af4fb9ca 100644 --- a/internal/application/repository/retriever/sqlite/repository.go +++ b/internal/application/repository/retriever/sqlite/repository.go @@ -60,13 +60,44 @@ func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRep } func initFTS5(db *gorm.DB) { - sql := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5( - content, source_id, chunk_id, knowledge_id, knowledge_base_id, - content='lite_embeddings', content_rowid='id', - tokenize='unicode61' - )` - if err := db.Exec(sql).Error; err != nil { - logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err) + var sqlStr string + db.Raw("SELECT sql FROM sqlite_master WHERE type='table' AND name='lite_embeddings_fts'").Scan(&sqlStr) + // If the table exists but is not contentless (i.e. uses content='lite_embeddings'), recreate it + if sqlStr != "" && strings.Contains(sqlStr, "content='lite_embeddings'") { + logger.GetLogger(context.Background()).Infof("[SQLite] Migrating FTS5 table to contentless table with manual bigram tokenization") + db.Exec("DROP TABLE IF EXISTS lite_embeddings_fts") + sqlStr = "" // Trigger recreate + } + + if sqlStr == "" { + createSQL := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5( + content, source_id, chunk_id, knowledge_id, knowledge_base_id, + content='', + contentless_delete=1, + tokenize='unicode61' + )` + if err := db.Exec(createSQL).Error; err != nil { + logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err) + return + } + logger.GetLogger(context.Background()).Infof("[SQLite] Populating contentless FTS5 table from lite_embeddings with bigrams") + + // To populate, we need to read all rows and insert them via Go to apply bigram tokenization + type Row struct { + ID uint + Content string + SourceID string + ChunkID string + KnowledgeID string + KnowledgeBaseID string + } + var rows []Row + db.Raw("SELECT id, content, source_id, chunk_id, knowledge_id, knowledge_base_id FROM lite_embeddings").Scan(&rows) + for _, r := range rows { + db.Exec(`INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) + VALUES(?, ?, ?, ?, ?, ?)`, + r.ID, tokenizeCJKBigram(r.Content), r.SourceID, r.ChunkID, r.KnowledgeID, r.KnowledgeBaseID) + } } } @@ -276,7 +307,7 @@ func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.Re SELECT e.id, e.source_id, e.source_type, e.chunk_id, e.knowledge_id, e.knowledge_base_id, e.tag_id, e.content, - bm25(lite_embeddings_fts) AS score + (bm25(lite_embeddings_fts) * -1000000.0) AS score FROM lite_embeddings_fts JOIN lite_embeddings e ON e.id = lite_embeddings_fts.rowid WHERE lite_embeddings_fts MATCH ? @@ -290,7 +321,7 @@ func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.Re args = append(args, wp.args...) } - sql += " ORDER BY score ASC LIMIT ?" + sql += " ORDER BY score DESC LIMIT ?" args = append(args, params.TopK) type ftsResult struct { @@ -315,11 +346,12 @@ func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.Re items := make([]*types.IndexWithScore, len(rows)) for i, row := range rows { - // bm25 越小越相关 → 转成正向分数 - score := -row.Score + // bm25 is originally negative and very small, we multiplied it by -1000000.0 in SQL + // to make it positive and human-readable. + score := row.Score - logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, bm25_raw=%.4f, score=%.4f, content_preview=%.60s", - i+1, row.ChunkID, row.Score, score, row.Content) + logger.GetLogger(ctx).Infof("[SQLite] keywordsRetrieve: #%d chunk_id=%s, score=%.4f, content_preview=%.60s", + i+1, row.ChunkID, score, row.Content) items[i] = &types.IndexWithScore{ ID: fmt.Sprintf("%d", row.ID), @@ -488,6 +520,9 @@ func (r *sqliteRepository) deleteRowsAndVecs(_ context.Context, rows []sqliteEmb r.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id) } } + for _, row := range rows { + r.db.Exec("DELETE FROM lite_embeddings_fts WHERE rowid = ?", row.ID) + } } func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) { @@ -505,8 +540,9 @@ func (r *sqliteRepository) syncFTS5Insert(_ context.Context, row *sqliteEmbeddin if row.ID == 0 { return } + tokenizedContent := tokenizeCJKBigram(row.Content) sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)` - r.db.Exec(sql, row.ID, row.Content, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID) + r.db.Exec(sql, row.ID, tokenizedContent, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID) } type whereClause struct { @@ -553,61 +589,77 @@ func toInterfaceSlice(ss []string) []interface{} { return out } -// sanitizeFTS5Query builds an FTS5 query from user input. -// CJK characters are split into overlapping bigrams (mimicking CJK analyzers used -// by Elasticsearch etc.) and joined with OR for broad partial matching. -// Non-CJK words are kept intact. BM25 ranking naturally boosts documents -// that match more tokens. +// tokenizeCJKBigram splits continuous CJK character sequences into overlapping bigrams. +// Non-CJK words are kept intact. This approach maximizes recall for CJK search. +func tokenizeCJKBigram(text string) string { + var parts []string + var currentCJK []rune + var currentNonCJK strings.Builder + + flushCJK := func() { + if len(currentCJK) == 0 { + return + } + if len(currentCJK) == 1 { + parts = append(parts, string(currentCJK[0])) + } else { + for i := 0; i < len(currentCJK)-1; i++ { + parts = append(parts, string(currentCJK[i])+string(currentCJK[i+1])) + } + } + currentCJK = currentCJK[:0] + } + + flushNonCJK := func() { + if currentNonCJK.Len() > 0 { + parts = append(parts, currentNonCJK.String()) + currentNonCJK.Reset() + } + } + + for _, r := range text { + // unicode.Han covers Chinese characters + if unicode.Is(unicode.Han, r) { + flushNonCJK() + currentCJK = append(currentCJK, r) + } else if unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r) { + // Delimiters + flushCJK() + flushNonCJK() + } else { + // Alphanumeric or other languages + flushCJK() + currentNonCJK.WriteRune(r) + } + } + flushCJK() + flushNonCJK() + + return strings.Join(parts, " ") +} + +// sanitizeFTS5Query builds an FTS5 query from user input by applying bigram tokenization. func sanitizeFTS5Query(q string) string { q = strings.TrimSpace(q) if q == "" { return q } - var cjkRunes []rune - var nonCJKWords []string - var buf strings.Builder - - flushNonCJK := func() { - if buf.Len() > 0 { - nonCJKWords = append(nonCJKWords, buf.String()) - buf.Reset() - } - } - - for _, r := range q { - if unicode.Is(unicode.Han, r) { - flushNonCJK() - cjkRunes = append(cjkRunes, r) - } else if unicode.IsSpace(r) || r == '|' { - flushNonCJK() - } else if r == '"' || r == '*' || r == '(' || r == ')' || r == '{' || r == '}' { - // skip FTS5 special characters - } else { - buf.WriteRune(r) - } - } - flushNonCJK() + // Tokenize input query with bigrams (same as during indexing) + tokenized := tokenizeCJKBigram(q) + fields := strings.Fields(tokenized) var parts []string - - // CJK bigrams for better matching (e.g. "苹果第四季度" → "苹果" OR "果第" OR "第四" OR ...) - if len(cjkRunes) == 1 { - parts = append(parts, `"`+string(cjkRunes[0])+`"`) - } else { - for i := 0; i < len(cjkRunes)-1; i++ { - bigram := string(cjkRunes[i]) + string(cjkRunes[i+1]) - parts = append(parts, `"`+bigram+`"`) + for _, f := range fields { + if f != "" { + parts = append(parts, `"`+f+`"`) } } - for _, w := range nonCJKWords { - w = strings.ReplaceAll(w, `"`, `""`) - parts = append(parts, `"`+w+`"`) - } - if len(parts) == 0 { - return `"` + strings.ReplaceAll(q, `"`, `""`) + `"` + return "" } + + // Use OR because we want fuzzy match across multiple bigrams/words return strings.Join(parts, " OR ") } diff --git a/internal/application/service/chat_pipeline/search.go b/internal/application/service/chat_pipeline/search.go index eb9b24d6..003e08c5 100644 --- a/internal/application/service/chat_pipeline/search.go +++ b/internal/application/service/chat_pipeline/search.go @@ -15,15 +15,15 @@ import ( // PluginSearch implements search functionality for chat pipeline type PluginSearch struct { - knowledgeBaseService interfaces.KnowledgeBaseService - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService - config *config.Config - webSearchService interfaces.WebSearchService - tenantService interfaces.TenantService - sessionService interfaces.SessionService - webSearchStateService interfaces.WebSearchStateService - webSearchProviderRepo interfaces.WebSearchProviderRepository + knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService + config *config.Config + webSearchService interfaces.WebSearchService + tenantService interfaces.TenantService + sessionService interfaces.SessionService + webSearchStateService interfaces.WebSearchStateService + webSearchProviderRepo interfaces.WebSearchProviderRepository } func NewPluginSearch(eventManager *EventManager, @@ -620,21 +620,34 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types return nil } tenant, _ := types.TenantInfoFromContext(ctx) - if tenant == nil || tenant.WebSearchConfig == nil { + providerID := chatManage.WebSearchProviderID + + var webConfig *types.WebSearchConfig + if tenant != nil && tenant.WebSearchConfig != nil { + // Clone tenant config so we can safely override MaxResults + cfg := *tenant.WebSearchConfig + webConfig = &cfg + } else if providerID != "" { + webConfig = &types.WebSearchConfig{ + MaxResults: 10, + } + } else { pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{ "tenant_id": chatManage.TenantID, }) return nil } - // Use provider ID already resolved by session layer (agent config > tenant default) - providerID := chatManage.WebSearchProviderID + // Apply agent-level web search overrides + if chatManage.WebSearchMaxResults > 0 { + webConfig.MaxResults = chatManage.WebSearchMaxResults + } pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{ "tenant_id": chatManage.TenantID, "provider_id": providerID, }) - webResults, err := p.webSearchService.Search(ctx, providerID, tenant.WebSearchConfig, chatManage.RewriteQuery) + webResults, err := p.webSearchService.Search(ctx, providerID, webConfig, chatManage.RewriteQuery) if err != nil { pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{ "tenant_id": chatManage.TenantID, @@ -643,22 +656,22 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types return nil } // Build questions using RewriteQuery only - questions := []string{strings.TrimSpace(chatManage.RewriteQuery)} + // questions := []string{strings.TrimSpace(chatManage.RewriteQuery)} // Load session-scoped temp KB state from Redis using WebSearchStateRepository - tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID) - compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG( - ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig, - p.knowledgeBaseService, p.knowledgeService, seen, ids, - ) - if err != nil { - pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{ - "error": err.Error(), - }) - } else { - webResults = compressed - // Persist temp KB state back into Redis using WebSearchStateRepository - p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs) - } + // tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID) + // compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG( + // ctx, chatManage.SessionID, tempKBID, questions, webResults, webConfig, + // p.knowledgeBaseService, p.knowledgeService, seen, ids, + // ) + // if err != nil { + // pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{ + // "error": err.Error(), + // }) + // } else { + // webResults = compressed + // // Persist temp KB state back into Redis using WebSearchStateRepository + // p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs) + // } res := searchutil.ConvertWebSearchResults(webResults) pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{ "hit_count": len(res), diff --git a/internal/application/service/session_knowledge_qa.go b/internal/application/service/session_knowledge_qa.go index 49ca2e7a..145b21ee 100644 --- a/internal/application/service/session_knowledge_qa.go +++ b/internal/application/service/session_knowledge_qa.go @@ -115,6 +115,7 @@ func (s *sessionService) KnowledgeQA( RewritePromptUser: s.cfg.Conversation.RewritePromptUser, WebSearchEnabled: req.WebSearchEnabled, WebSearchProviderID: s.resolveWebSearchProviderID(ctx, req, retrievalTenantID), + WebSearchMaxResults: s.resolveWebSearchMaxResults(ctx, req), WebFetchEnabled: s.resolveWebFetchEnabled(req), WebFetchTopN: s.resolveWebFetchTopN(req), TenantID: retrievalTenantID, @@ -841,3 +842,16 @@ func (s *sessionService) resolveWebFetchTopN(req *types.QARequest) int { } return 3 } + +// resolveWebSearchMaxResults returns the max results for web search. +// Priority: agent config > tenant default > default (10) +func (s *sessionService) resolveWebSearchMaxResults(ctx context.Context, req *types.QARequest) int { + if req.CustomAgent != nil && req.CustomAgent.Config.WebSearchMaxResults > 0 { + return req.CustomAgent.Config.WebSearchMaxResults + } + tenantInfo, _ := types.TenantInfoFromContext(ctx) + if tenantInfo != nil && tenantInfo.WebSearchConfig != nil && tenantInfo.WebSearchConfig.MaxResults > 0 { + return tenantInfo.WebSearchConfig.MaxResults + } + return 10 +} diff --git a/internal/application/service/session_qa_helpers.go b/internal/application/service/session_qa_helpers.go index b3fc2d5a..b7680b1c 100644 --- a/internal/application/service/session_qa_helpers.go +++ b/internal/application/service/session_qa_helpers.go @@ -176,6 +176,11 @@ func (s *sessionService) applyAgentOverridesToChatManage( cm.FallbackPrompt = customAgent.Config.FallbackPrompt } + // Override web search settings + if customAgent.Config.WebSearchMaxResults > 0 { + cm.WebSearchMaxResults = customAgent.Config.WebSearchMaxResults + } + // Override history turns if customAgent.Config.HistoryTurns > 0 { cm.MaxRounds = customAgent.Config.HistoryTurns diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go index 4adb03fa..e41cc2bc 100644 --- a/internal/application/service/web_search.go +++ b/internal/application/service/web_search.go @@ -7,8 +7,8 @@ import ( "strings" "time" - infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/config" + infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" @@ -216,8 +216,8 @@ func (s *WebSearchService) CompressWithRAG( for _, q := range questions { params := types.SearchParams{ QueryText: q, - VectorThreshold: 0.5, - KeywordThreshold: 0.5, + VectorThreshold: 0.2, + KeywordThreshold: 0.2, MatchCount: matchCount, } results, err := kbSvc.HybridSearch(ctx, tempKBID, params) diff --git a/internal/application/service/web_search_state.go b/internal/application/service/web_search_state.go index 430e133c..811458cb 100644 --- a/internal/application/service/web_search_state.go +++ b/internal/application/service/web_search_state.go @@ -36,6 +36,9 @@ func (s *webSearchStateService) GetWebSearchTempKBState( ctx context.Context, sessionID string, ) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) { + if s.redisClient == nil { + return "", make(map[string]bool), []string{} + } stateKey := fmt.Sprintf("tempkb:%s", sessionID) if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 { var state struct { @@ -65,6 +68,9 @@ func (s *webSearchStateService) SaveWebSearchTempKBState( seenURLs map[string]bool, knowledgeIDs []string, ) { + if s.redisClient == nil { + return + } stateKey := fmt.Sprintf("tempkb:%s", sessionID) state := struct { KBID string `json:"kbID"` diff --git a/internal/container/container.go b/internal/container/container.go index 30a019e9..18e478b7 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -355,6 +355,7 @@ func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, e func initDatabase(cfg *config.Config) (*gorm.DB, error) { var dialector gorm.Dialector var migrateDSN string + var sqliteDBPath string switch os.Getenv("DB_DRIVER") { case "postgres": // DSN for GORM (key-value format) @@ -412,6 +413,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { sqlite_vec.Auto() dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on" dialector = sqlite.Open(dsn) + sqliteDBPath = dbPath migrateDSN = "sqlite3://" + dbPath logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath) default: @@ -445,6 +447,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { autoRecover := os.Getenv("AUTO_RECOVER_DIRTY") != "false" migrationOpts := database.MigrationOptions{ AutoRecoverDirty: autoRecover, + SQLiteDBPath: sqliteDBPath, } // Run base migrations (all versioned migrations including embeddings) @@ -505,6 +508,56 @@ func resolveStorageProviderPending(db *gorm.DB) { } else if result.RowsAffected > 0 { logger.Infof(context.Background(), "Resolved %d knowledge bases with __pending_env__ storage provider → %s", result.RowsAffected, storageType) } + + // Reset any pending tasks left over from previous aborted runs (Lite App mode) + resetPendingTasks(db) +} + +// resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing +// due to an unexpected application restart when using in-memory queues (Lite mode). +func resetPendingTasks(db *gorm.DB) { + if os.Getenv("REDIS_ADDR") != "" { + return // Distributed queue (Asynq) will handle its own retries + } + + // 1. Reset knowledge parsing tasks + result := db.Model(&types.Knowledge{}). + Where("parse_status IN ?", []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusDeleting}). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": "Task interrupted due to application restart", + }) + if result.Error != nil { + logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error) + } else if result.RowsAffected > 0 { + logger.Infof(context.Background(), "Reset %d stuck knowledge parsing tasks to failed state", result.RowsAffected) + } + + // 2. Reset knowledge summary tasks + resultSummary := db.Model(&types.Knowledge{}). + Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing}). + Updates(map[string]interface{}{ + "summary_status": types.SummaryStatusFailed, + }) + if resultSummary.Error != nil { + logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error) + } else if resultSummary.RowsAffected > 0 { + logger.Infof(context.Background(), "Reset %d stuck summary generation tasks to failed state", resultSummary.RowsAffected) + } + + // 3. Reset data source sync tasks + resultSync := db.Model(&types.SyncLog{}). + Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}). + Updates(map[string]interface{}{ + "status": types.SyncLogStatusFailed, + "error_message": "Sync interrupted due to application restart", + "end_time": time.Now(), + }) + if resultSync.Error != nil { + logger.Warnf(context.Background(), "Failed to reset pending data source sync tasks: %v", resultSync.Error) + } else if resultSync.RowsAffected > 0 { + logger.Infof(context.Background(), "Reset %d stuck data source sync tasks to failed state", resultSync.RowsAffected) + } } // initFileService initializes file storage service diff --git a/internal/database/migration.go b/internal/database/migration.go index 5c836b2e..a57108ea 100644 --- a/internal/database/migration.go +++ b/internal/database/migration.go @@ -2,6 +2,7 @@ package database import ( "context" + "database/sql" "fmt" "os" "strings" @@ -10,7 +11,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/golang-migrate/migrate/v4" _ "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/golang-migrate/migrate/v4/database/sqlite3" + sqlite3migrate "github.com/golang-migrate/migrate/v4/database/sqlite3" _ "github.com/golang-migrate/migrate/v4/source/file" ) @@ -46,6 +47,12 @@ type MigrationOptions struct { // AutoRecoverDirty when true, automatically attempts to recover from dirty state // by forcing to the previous version and retrying the migration AutoRecoverDirty bool + + // SQLiteDBPath is the raw filesystem path to the SQLite database file. + // When set, the migrator opens the DB directly via sql.Open instead of + // parsing a URL-based DSN, which avoids breakage when the path contains + // spaces (e.g. macOS "Application Support"). + SQLiteDBPath string } // RunMigrationsWithOptions executes all pending database migrations with custom options @@ -59,10 +66,31 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { migrationsPath = "file://migrations/sqlite" } - m, err := migrate.New(migrationsPath, dsn) - if err != nil { - logger.Errorf(ctx, "Failed to create migrate instance: %v", err) - return fmt.Errorf("failed to create migrate instance: %w", err) + var m *migrate.Migrate + if opts.SQLiteDBPath != "" { + sqlDB, err := sql.Open("sqlite3", opts.SQLiteDBPath) + if err != nil { + logger.Errorf(ctx, "Failed to open sqlite db for migration: %v", err) + return fmt.Errorf("failed to open sqlite db for migration: %w", err) + } + driver, err := sqlite3migrate.WithInstance(sqlDB, &sqlite3migrate.Config{}) + if err != nil { + sqlDB.Close() + logger.Errorf(ctx, "Failed to create sqlite3 migrate driver: %v", err) + return fmt.Errorf("failed to create sqlite3 migrate driver: %w", err) + } + m, err = migrate.NewWithDatabaseInstance(migrationsPath, "sqlite3", driver) + if err != nil { + logger.Errorf(ctx, "Failed to create migrate instance: %v", err) + return fmt.Errorf("failed to create migrate instance: %w", err) + } + } else { + var err error + m, err = migrate.New(migrationsPath, dsn) + if err != nil { + logger.Errorf(ctx, "Failed to create migrate instance: %v", err) + return fmt.Errorf("failed to create migrate instance: %w", err) + } } defer m.Close() diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.go index 42a8ab8d..487b3a8e 100644 --- a/internal/datasource/connector/notion/connector.go +++ b/internal/datasource/connector/notion/connector.go @@ -394,9 +394,6 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id if err != nil || len(records) == 0 { return nil } - // Same database may be reached via two different IDs (data_source_id from search - // vs database_container_id from child_database blocks). Mark the canonical - // data_source_id as visited so the second path is deduped. if queryID != "" && queryID != id { if visited[queryID] { return nil @@ -404,18 +401,15 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id visited[queryID] = true } - propNames := extractPropertySchema(records[0]) - var items []types.FetchedItem for _, record := range records { - if record.InTrash { - continue - } - item := c.buildRecordItem(ctx, client, record, propNames, dbTitle) - if item != nil { - items = append(items, *item) - } + visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls } - return items + + item := c.buildDatabaseItem(ctx, client, id, dbTitle, records) + if item != nil { + return []types.FetchedItem{*item} + } + return nil } // fetchDatabaseIncremental syncs only changed records by comparing edit times against cursor. @@ -438,32 +432,33 @@ func (c *Connector) fetchDatabaseIncremental(ctx context.Context, client *notion } recordEditTimes := make(map[string]time.Time, len(records)) - var propNames []string - if len(records) > 0 { - propNames = extractPropertySchema(records[0]) - } - - var items []types.FetchedItem changedCount := 0 + for _, record := range records { + visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls + if record.InTrash { continue } recordEditTimes[record.ID] = record.LastEditedTime prevTime, existed := prevEditTimes[record.ID] - if existed && record.LastEditedTime.Equal(prevTime) { - continue - } - changedCount++ - item := c.buildRecordItem(ctx, client, record, propNames, dbTitle) - if item != nil { - items = append(items, *item) + if !existed || !record.LastEditedTime.Equal(prevTime) { + changedCount++ } } logger.Infof(ctx, "[Notion] database %s incremental: %d changed out of %d records", id, changedCount, len(records)) - return items, recordEditTimes + + // Rebuild the entire database table if any record changed, or if we don't have previous times (first sync) + if changedCount > 0 || len(prevEditTimes) == 0 { + item := c.buildDatabaseItem(ctx, client, id, dbTitle, records) + if item != nil { + return []types.FetchedItem{*item}, recordEditTimes + } + } + + return nil, recordEditTimes } // queryDatabaseRecords resolves the database ID and queries all records. @@ -545,6 +540,103 @@ func (c *Connector) buildRecordItem(ctx context.Context, client *notionClient, r } } +// buildDatabaseItem converts a database into a single Markdown table document. +func (c *Connector) buildDatabaseItem(ctx context.Context, client *notionClient, id string, dbTitle string, records []notionPage) *types.FetchedItem { + if len(records) == 0 { + return nil + } + + propNames := extractPropertySchema(records[0]) + + var content strings.Builder + title := dbTitle + if title == "" { + title = defaultUntitledName + } + content.WriteString("# " + title + "\n\n") + + // Table header + content.WriteString("| Title ") + for _, name := range propNames { + content.WriteString("| " + strings.ReplaceAll(name, "|", "\\|") + " ") + } + content.WriteString("|\n|") + content.WriteString("---|") + for range propNames { + content.WriteString("---|") + } + content.WriteString("\n") + + var extraContent strings.Builder + + // Table rows + for _, record := range records { + if record.InTrash { + continue + } + + recordTitle := record.Title + if recordTitle == "" { + recordTitle = defaultUntitledName + } + + content.WriteString("| " + strings.ReplaceAll(recordTitle, "|", "\\|") + " ") + + if record.RawProperties != nil { + var props map[string]interface{} + json.Unmarshal(record.RawProperties, &props) + for _, name := range propNames { + val := "" + if propMap, ok := props[name].(map[string]interface{}); ok { + val = propertyToString(propMap) + } + val = strings.ReplaceAll(val, "\n", "
") + val = strings.ReplaceAll(val, "|", "\\|") + content.WriteString("| " + val + " ") + } + } + content.WriteString("|\n") + + // Fetch blocks for the record, in case it has page content + blocks, err := client.GetBlockChildrenAll(ctx, record.ID) + if err == nil && len(blocks) > 0 { + resolveFileUploads(ctx, client, blocks) + markdown, _ := BlocksToMarkdown(blocks) + if strings.TrimSpace(markdown) != "" { + extraContent.WriteString("\n## " + recordTitle + " 内容\n\n" + markdown + "\n") + } + } + } + + if extraContent.Len() > 0 { + content.WriteString("\n" + extraContent.String()) + } + + bodyStr := strings.TrimSpace(content.String()) + if bodyStr == "" { + return nil + } + + updatedAt := time.Now() + if len(records) > 0 { + updatedAt = records[0].LastEditedTime + } + + return &types.FetchedItem{ + ExternalID: id, + Title: title, + Content: []byte(bodyStr), + ContentType: contentTypeMarkdown, + FileName: title + ".md", + URL: fmt.Sprintf("https://notion.so/%s", strings.ReplaceAll(id, "-", "")), + UpdatedAt: updatedAt, + Metadata: map[string]string{ + "channel": types.ChannelNotion, + "object_type": objectTypeDatabase, + }, + } +} + // discoverAllResources uses the Search API to find all pages and data_sources, // then filters by parent chain via BFS from the selected resource IDs. // Returns the included pages plus the excluded set (visible pages NOT under any diff --git a/internal/datasource/connector/notion/connector_test.go b/internal/datasource/connector/notion/connector_test.go index 1c21f410..5846e085 100644 --- a/internal/datasource/connector/notion/connector_test.go +++ b/internal/datasource/connector/notion/connector_test.go @@ -2,6 +2,7 @@ package notion import ( "context" + "strings" "testing" "github.com/Tencent/WeKnora/internal/types" @@ -121,16 +122,13 @@ func TestConnectorFetchAll_Database(t *testing.T) { t.Fatal("expected at least 1 item") } - // Each database record should be synced as an individual knowledge item + // The entire database should be synced as a single table knowledge item found := false for _, item := range items { - if item.ExternalID == "record-1" { + if item.ExternalID == "db-1" { found = true - if item.Metadata["object_type"] != "page" { - t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "page") - } - if item.Metadata["database"] != "Test Database" { - t.Errorf("database = %q, want %q", item.Metadata["database"], "Test Database") + if item.Metadata["object_type"] != "database" { + t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "database") } if item.ContentType != "text/markdown" { t.Errorf("ContentType = %q", item.ContentType) @@ -138,10 +136,15 @@ func TestConnectorFetchAll_Database(t *testing.T) { if len(item.Content) == 0 { t.Error("expected non-empty content") } + // Verify it contains table headers + contentStr := string(item.Content) + if !strings.Contains(contentStr, "| Title |") { + t.Errorf("expected table header in content, got: %s", contentStr) + } } } if !found { - t.Error("record-1 not found in items") + t.Error("db-1 not found in items") } } diff --git a/internal/datasource/connector/notion/markdown_test.go b/internal/datasource/connector/notion/markdown_test.go index ad6b4c0c..9a256e35 100644 --- a/internal/datasource/connector/notion/markdown_test.go +++ b/internal/datasource/connector/notion/markdown_test.go @@ -134,7 +134,7 @@ func TestBlocksToMarkdown_Paragraph(t *testing.T) { }), } md, attachments := BlocksToMarkdown(blocks) - if md != "Hello world\n\n" { + if md != "Hello world\n" { t.Errorf("markdown = %q", md) } if len(attachments) != 0 { @@ -161,7 +161,7 @@ func TestBlocksToMarkdown_Headings(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "# H1\n\n## H2\n\n### H3\n\n" + expected := "# H1\n\n## H2\n\n### H3\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -177,7 +177,7 @@ func TestBlocksToMarkdown_Code(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "```go\nfmt.Println()\n```\n\n" + expected := "```go\nfmt.Println()\n```\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -197,7 +197,7 @@ func TestBlocksToMarkdown_List(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "- Item 1\n- Item 2\n\n" + expected := "- Item 1\n- Item 2\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -219,7 +219,7 @@ func TestBlocksToMarkdown_ToDo(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "- [ ] Unchecked\n- [x] Checked\n\n" + expected := "- [ ] Unchecked\n- [x] Checked\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -234,7 +234,7 @@ func TestBlocksToMarkdown_Quote(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "> A wise quote\n\n" + expected := "> A wise quote\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -245,7 +245,7 @@ func TestBlocksToMarkdown_Divider(t *testing.T) { {ID: "blk-div", Type: "divider"}, } md, _ := BlocksToMarkdown(blocks) - if md != "---\n\n" { + if md != "---\n" { t.Errorf("markdown = %q", md) } } @@ -257,7 +257,7 @@ func TestBlocksToMarkdown_Equation(t *testing.T) { }), } md, _ := BlocksToMarkdown(blocks) - expected := "$$E = mc^2$$\n\n" + expected := "$$E = mc^2$$\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -277,7 +277,7 @@ func TestBlocksToMarkdown_Image(t *testing.T) { }), } md, attachments := BlocksToMarkdown(blocks) - if md != "![A photo](https://s3.example.com/img.png)\n\n" { + if md != "![A photo](https://s3.example.com/img.png)\n" { t.Errorf("markdown = %q", md) } if len(attachments) != 1 { @@ -296,7 +296,7 @@ func TestBlocksToMarkdown_UnknownType(t *testing.T) { {ID: "blk-unknown", Type: "future_block_type"}, } md, _ := BlocksToMarkdown(blocks) - if md != "" { + if md != "\n" { t.Errorf("unknown block should produce empty output, got %q", md) } } @@ -323,7 +323,7 @@ func TestBlocksToMarkdown_Table(t *testing.T) { }, } md, _ := BlocksToMarkdown([]notionBlock{tableBlock}) - expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\n" + expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } @@ -345,7 +345,7 @@ func TestBlocksToMarkdown_NestedList(t *testing.T) { } md, _ := BlocksToMarkdown([]notionBlock{parent}) - expected := "- Parent\n - Child\n\n" + expected := "- Parent\n - Child\n" if md != expected { t.Errorf("markdown = %q, want %q", md, expected) } diff --git a/internal/im/wecom/longconn.go b/internal/im/wecom/longconn.go index 20b236e1..e9ead313 100644 --- a/internal/im/wecom/longconn.go +++ b/internal/im/wecom/longconn.go @@ -286,6 +286,7 @@ func (c *LongConnClient) SendStreamChunk(ctx context.Context, incoming *im.Incom } // EndStream sends the final frame with the full accumulated content and cleans up. +// It retries briefly if the connection is temporarily unavailable during a reconnect. func (c *LongConnClient) EndStream(ctx context.Context, incoming *im.IncomingMessage, streamID string) error { c.streamBufsMu.Lock() buf, ok := c.streamBufs[streamID] @@ -296,7 +297,23 @@ func (c *LongConnClient) EndStream(ctx context.Context, incoming *im.IncomingMes } c.streamBufsMu.Unlock() - return c.sendStreamFrame(incoming, streamID, fullContent, true) + err := c.sendStreamFrame(incoming, streamID, fullContent, true) + if err == nil { + return nil + } + + // Retry up to 3 times with short delays to ride out a reconnection window. + for i := 0; i < 3; i++ { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + if retryErr := c.sendStreamFrame(incoming, streamID, fullContent, true); retryErr == nil { + return nil + } + } + return err } func (c *LongConnClient) sendStreamFrame(incoming *im.IncomingMessage, streamID, content string, finish bool) error { @@ -342,12 +359,11 @@ func (c *LongConnClient) connectAndRun(ctx context.Context) error { c.conn = nil c.mu.Unlock() _ = conn.Close() - - // Clear in-flight stream buffers to prevent memory leaks on reconnect. - // Streams interrupted by a connection drop cannot be resumed. - c.streamBufsMu.Lock() - c.streamBufs = nil - c.streamBufsMu.Unlock() + // NOTE: streamBufs is intentionally NOT cleared on reconnect. + // Active streams survive reconnections — the WeCom replace-based + // protocol means the next SendStreamChunk will resend the full + // accumulated content on the new connection. EndStream always + // cleans up the buffer, so there is no memory leak. }() // Authenticate diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 3eb08ee4..a1ad7f4f 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -3,11 +3,14 @@ package logger import ( "context" "fmt" + "io" "os" "path" + "path/filepath" "runtime" "sort" "strings" + "sync" "github.com/Tencent/WeKnora/internal/types" "github.com/sirupsen/logrus" @@ -16,6 +19,11 @@ import ( // appLogger 使用私有实例，避免外部依赖改写 logrus 全局状态导致日志丢失 var appLogger = logrus.New() +var ( + loggerMu sync.Mutex + activeLogFile *os.File +) + // LogLevel 日志级别类型 type LogLevel string @@ -130,12 +138,38 @@ func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) { // 初始化全局日志设置 func init() { + ConfigureFromEnv() +} + +// ConfigureFromEnv 重新从环境变量应用日志配置。 +// 这允许在 main() 中加载 .env 后，让 LOG_LEVEL / LOG_PATH 立即生效。 +func ConfigureFromEnv() { + loggerMu.Lock() + defer loggerMu.Unlock() + + if activeLogFile != nil { + _ = activeLogFile.Close() + activeLogFile = nil + } + // 根据环境变量设置全局日志级别 logLevel := getLogLevelFromEnv() appLogger.SetLevel(logLevel) - // 统一输出到 stdout，确保在 Docker 容器中与 GORM/GIN 日志合并展示 - appLogger.SetOutput(os.Stdout) + writer := io.Writer(os.Stdout) + logPath := resolveLogPathFromEnv() + if logPath != "" { + file, err := openLogFile(logPath) + if err != nil { + fmt.Fprintf(os.Stderr, "logger: failed to open log file %s: %v\n", logPath, err) + } else { + activeLogFile = file + writer = io.MultiWriter(os.Stdout, file) + } + } + + // 默认继续输出到 stdout，同时在可用时落盘到文件 + appLogger.SetOutput(writer) // 非终端（如 Docker 日志采集）禁用 ANSI 颜色，避免日志聚合/检索异常 forceColor := false @@ -199,6 +233,45 @@ func getLogLevelFromEnv() logrus.Level { } } +func resolveLogPathFromEnv() string { + if logPath := strings.TrimSpace(os.Getenv("LOG_PATH")); logPath != "" { + return filepath.Clean(logPath) + } + return defaultMacAppLogPath() +} + +func defaultMacAppLogPath() string { + execPath, err := os.Executable() + if err != nil || !strings.Contains(execPath, ".app/Contents/MacOS") { + return "" + } + + homeDir, err := os.UserHomeDir() + if err != nil { + return "" + } + + appName := "WeKnora Lite" + if idx := strings.Index(execPath, ".app/Contents/MacOS"); idx >= 0 { + bundleName := filepath.Base(execPath[:idx+4]) + if trimmed := strings.TrimSuffix(bundleName, ".app"); trimmed != "" { + appName = trimmed + } + } + + return filepath.Join(homeDir, "Library", "Logs", appName, appName+".log") +} + +func openLogFile(logPath string) (*os.File, error) { + dir := filepath.Dir(logPath) + if dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + } + return os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) +} + // 添加调用者字段 func addCaller(entry *logrus.Entry, skip int) *logrus.Entry { pc, file, line, ok := runtime.Caller(skip) diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.go index 10dca6cf..1e8a0265 100644 --- a/internal/types/chat_manage.go +++ b/internal/types/chat_manage.go @@ -46,12 +46,13 @@ type PipelineRequest struct { ChatModelSupportsVision bool `json:"-"` // Misc request-scoped config - TenantID uint64 `json:"-"` - WebSearchEnabled bool `json:"-"` - WebSearchProviderID string `json:"-"` // Resolved from agent config or tenant default - WebFetchEnabled bool `json:"-"` // Auto-fetch full page content for web search results after rerank - WebFetchTopN int `json:"-"` // Max pages to fetch (default 3) - Language string `json:"-"` + TenantID uint64 `json:"-"` + WebSearchEnabled bool `json:"-"` + WebSearchProviderID string `json:"-"` // Resolved from agent config or tenant default + WebSearchMaxResults int `json:"-"` // Resolved from agent config or tenant default + WebFetchEnabled bool `json:"-"` // Auto-fetch full page content for web search results after rerank + WebFetchTopN int `json:"-"` // Max pages to fetch (default 3) + Language string `json:"-"` } // QueryIntent represents the classified intent of a user query. @@ -187,9 +188,10 @@ func (c *ChatManage) Clone() *ChatManage { ChatModelSupportsVision: c.ChatModelSupportsVision, TenantID: c.TenantID, WebSearchEnabled: c.WebSearchEnabled, - WebSearchProviderID: c.WebSearchProviderID, - WebFetchEnabled: c.WebFetchEnabled, - WebFetchTopN: c.WebFetchTopN, + WebSearchProviderID: c.WebSearchProviderID, + WebSearchMaxResults: c.WebSearchMaxResults, + WebFetchEnabled: c.WebFetchEnabled, + WebFetchTopN: c.WebFetchTopN, Language: c.Language, }, PipelineState: PipelineState{ diff --git a/internal/types/json.go b/internal/types/json.go index e2aae2db..98756520 100644 --- a/internal/types/json.go +++ b/internal/types/json.go @@ -12,9 +12,24 @@ type JSON json.RawMessage // Scan implements the sql.Scanner interface. func (j *JSON) Scan(value interface{}) error { - bytes, ok := value.([]byte) - if !ok { - return errors.New("type assertion to []byte failed") + if value == nil { + *j = nil + return nil + } + + var bytes []byte + switch v := value.(type) { + case []byte: + bytes = v + case string: + bytes = []byte(v) + default: + return errors.New("type assertion to []byte or string failed") + } + + if len(bytes) == 0 { + *j = nil + return nil } result := json.RawMessage{}