feat(datasource): add RSS feed connector with full-text ingestion
Enable private RSS feeds as a knowledge base data source with custom headers, article body fetching, and Markdown output for incremental sync.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" role="img" aria-label="RSS">
|
||||
<circle cx="14" cy="34" r="4" fill="#FF6600"/>
|
||||
<path d="M10 22a18 18 0 0 1 18 18" fill="none" stroke="#FF6600" stroke-width="4" stroke-linecap="round"/>
|
||||
<path d="M10 14a26 26 0 0 1 26 26" fill="none" stroke="#FF6600" stroke-width="4" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 361 B |
@@ -7,9 +7,9 @@
|
||||
<div class="setting-drawer-resize-line" />
|
||||
</div>
|
||||
</teleport>
|
||||
<t-drawer v-model:visible="drawerVisible" :size="effectiveWidth" :z-index="2500" placement="right"
|
||||
<t-drawer v-model:visible="drawerVisible" v-bind="drawerPassthroughAttrs" :size="effectiveWidth" :z-index="2500" placement="right"
|
||||
attach="body" destroy-on-close
|
||||
:class="['setting-drawer', { 'setting-drawer--resizing': drawerResizing }]">
|
||||
:class="drawerClass">
|
||||
<!--
|
||||
Custom header. We replace TDesign's default header so we can put a leading
|
||||
icon badge and an optional subtitle (description) right next to the title,
|
||||
@@ -55,7 +55,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, onMounted, onUnmounted } from 'vue'
|
||||
import { ref, computed, useAttrs, onMounted, onUnmounted } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
interface Props {
|
||||
@@ -90,6 +90,8 @@ interface Props {
|
||||
hideFooter?: boolean
|
||||
}
|
||||
|
||||
defineOptions({ inheritAttrs: false })
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
description: '',
|
||||
icon: '',
|
||||
@@ -112,6 +114,12 @@ const emit = defineEmits<{
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
const attrs = useAttrs()
|
||||
|
||||
const drawerPassthroughAttrs = computed(() => {
|
||||
const { class: _class, ...rest } = attrs
|
||||
return rest
|
||||
})
|
||||
|
||||
// ---------- visibility ----------
|
||||
const drawerVisible = computed({
|
||||
@@ -172,6 +180,13 @@ const persistWidth = (width: number) => {
|
||||
|
||||
// ---------- Custom drag-resize (visible handle, same as doc-content) ----------
|
||||
const drawerResizing = ref(false)
|
||||
|
||||
const drawerClass = computed(() => [
|
||||
'setting-drawer',
|
||||
attrs.class,
|
||||
{ 'setting-drawer--resizing': drawerResizing.value },
|
||||
])
|
||||
|
||||
let resizeStartX = 0
|
||||
let resizeStartWidth = 0
|
||||
|
||||
|
||||
@@ -5418,6 +5418,7 @@ export default {
|
||||
editTitle: 'Edit Data Source',
|
||||
nameLabel: 'Name',
|
||||
namePlaceholder: 'Enter data source name',
|
||||
sectionBasic: 'Basic information',
|
||||
testConnection: 'Test Connection',
|
||||
testSuccess: 'Connection successful',
|
||||
testFailed: 'Connection failed',
|
||||
@@ -5493,11 +5494,13 @@ export default {
|
||||
feishu: 'Feishu',
|
||||
notion: 'Notion',
|
||||
yuque: 'Yuque',
|
||||
rss: 'RSS / Atom Feed',
|
||||
},
|
||||
connectorDesc: {
|
||||
feishu: 'Sync documents, spreadsheets and files from Feishu Wiki',
|
||||
notion: 'Sync pages and databases from Notion',
|
||||
yuque: 'Sync documents from Yuque knowledge bases',
|
||||
rss: 'Sync articles from RSS / Atom feeds',
|
||||
},
|
||||
field: {
|
||||
appId: 'App ID',
|
||||
@@ -5506,6 +5509,10 @@ export default {
|
||||
apiToken: 'API Token',
|
||||
baseUrl: 'Base URL (optional)',
|
||||
baseUrlHint: 'Leave empty to use the Yuque public cloud (https://www.yuque.com). For Yuque Enterprise or self-hosted deployments, enter your company domain (e.g. https://your-company.yuque.com).',
|
||||
feedUrls: 'Feed URLs',
|
||||
feedUrlsHint: 'One RSS / Atom feed URL per line; multiple feeds are supported.',
|
||||
authHeaders: 'Custom headers (optional)',
|
||||
authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx',
|
||||
},
|
||||
comingSoon: 'Coming soon',
|
||||
docHint: 'Get credentials at:',
|
||||
|
||||
@@ -5430,6 +5430,7 @@ export default {
|
||||
editTitle: "데이터 소스 편집",
|
||||
nameLabel: "이름",
|
||||
namePlaceholder: "데이터 소스 이름 입력",
|
||||
sectionBasic: "기본 정보",
|
||||
testConnection: "연결 테스트",
|
||||
testSuccess: "연결 성공",
|
||||
testFailed: "연결 실패",
|
||||
@@ -5505,11 +5506,13 @@ export default {
|
||||
feishu: "페이슈 (Feishu)",
|
||||
notion: "Notion",
|
||||
yuque: "위큐 (Yuque)",
|
||||
rss: "RSS / Atom 피드",
|
||||
},
|
||||
connectorDesc: {
|
||||
feishu: "페이슈 위키에서 문서, 스프레드시트, 파일 동기화",
|
||||
notion: "Notion에서 페이지 및 데이터베이스 동기화",
|
||||
yuque: "위큐 지식베이스에서 문서 동기화",
|
||||
rss: "RSS / Atom 피드에서 글 동기화",
|
||||
},
|
||||
field: {
|
||||
appId: "App ID",
|
||||
@@ -5518,6 +5521,10 @@ export default {
|
||||
baseUrl: "Base URL",
|
||||
apiToken: "API Token",
|
||||
baseUrlHint: "비워두면 Yuque 퍼블릭 클라우드 https://www.yuque.com 를 사용합니다. Yuque Enterprise 또는 사설 배포를 사용하는 경우 기업 도메인(예: https://your-company.yuque.com)을 입력하세요",
|
||||
feedUrls: "피드 주소",
|
||||
feedUrlsHint: "한 줄에 하나씩 RSS / Atom 피드 주소를 입력하세요. 여러 개를 함께 입력할 수 있습니다.",
|
||||
authHeaders: "사용자 지정 헤더 (선택)",
|
||||
authHeadersHint: "비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx",
|
||||
},
|
||||
comingSoon: "곧 지원 예정",
|
||||
docHint: "다음에서 인증 정보를 받으세요:",
|
||||
|
||||
@@ -5252,6 +5252,7 @@ export default {
|
||||
editTitle: 'Редактировать источник данных',
|
||||
nameLabel: 'Название',
|
||||
namePlaceholder: 'Введите название источника данных',
|
||||
sectionBasic: 'Основная информация',
|
||||
testConnection: 'Проверить подключение',
|
||||
testSuccess: 'Подключение успешно',
|
||||
testFailed: 'Подключение не удалось',
|
||||
@@ -5327,11 +5328,13 @@ export default {
|
||||
feishu: 'Feishu (Фэйшу)',
|
||||
notion: 'Notion',
|
||||
yuque: 'Yuque (Юйцюэ)',
|
||||
rss: 'RSS / Atom лента',
|
||||
},
|
||||
connectorDesc: {
|
||||
feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki',
|
||||
notion: 'Синхронизация страниц и баз данных из Notion',
|
||||
yuque: 'Синхронизация документов из баз знаний Yuque',
|
||||
rss: 'Синхронизация статей из лент RSS / Atom',
|
||||
},
|
||||
field: {
|
||||
appId: 'App ID',
|
||||
@@ -5340,6 +5343,10 @@ export default {
|
||||
baseUrl: 'Base URL',
|
||||
apiToken: 'API Token',
|
||||
baseUrlHint: 'Оставьте пустым, чтобы использовать публичное облако Yuque https://www.yuque.com. Если вы используете Yuque Enterprise или приватное развёртывание, укажите корпоративный домен (например, https://your-company.yuque.com)',
|
||||
feedUrls: 'Адреса лент',
|
||||
feedUrlsHint: 'По одному адресу ленты RSS / Atom в строке; можно указать несколько.',
|
||||
authHeaders: 'Пользовательские заголовки (необязательно)',
|
||||
authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx',
|
||||
},
|
||||
comingSoon: 'Скоро',
|
||||
docHint: 'Получить учётные данные можно здесь:',
|
||||
|
||||
@@ -5436,6 +5436,7 @@ export default {
|
||||
editTitle: "编辑数据源",
|
||||
nameLabel: "名称",
|
||||
namePlaceholder: "输入数据源名称",
|
||||
sectionBasic: "基本信息",
|
||||
testConnection: "测试连接",
|
||||
testSuccess: "连接成功",
|
||||
testFailed: "连接失败",
|
||||
@@ -5511,11 +5512,13 @@ export default {
|
||||
feishu: "飞书",
|
||||
notion: "Notion",
|
||||
yuque: "语雀",
|
||||
rss: "RSS / Atom 订阅",
|
||||
},
|
||||
connectorDesc: {
|
||||
feishu: "同步飞书知识库中的文档、表格、文件",
|
||||
notion: "同步 Notion 中的页面和数据库",
|
||||
yuque: "同步语雀知识库中的文档",
|
||||
rss: "同步 RSS / Atom 订阅源中的文章",
|
||||
},
|
||||
field: {
|
||||
appId: "App ID",
|
||||
@@ -5524,6 +5527,10 @@ export default {
|
||||
apiToken: "API Token",
|
||||
baseUrl: "Base URL(可选)",
|
||||
baseUrlHint: "留空将使用语雀公有云 https://www.yuque.com;如果你使用的是语雀企业版或私有部署,请填写企业域名(例如 https://your-company.yuque.com)",
|
||||
feedUrls: "订阅源地址",
|
||||
feedUrlsHint: "每行一个 RSS / Atom 订阅源地址,支持同时填写多个",
|
||||
authHeaders: "自定义请求头(可选)",
|
||||
authHeadersHint: "用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx",
|
||||
},
|
||||
comingSoon: "即将支持",
|
||||
docHint: "在以下地址获取凭证:",
|
||||
|
||||
@@ -3948,20 +3948,15 @@ const handleSave = async () => {
|
||||
|
||||
.nav-badge {
|
||||
flex-shrink: 0;
|
||||
min-width: 18px;
|
||||
margin-left: 2px;
|
||||
padding: 0 6px;
|
||||
border-radius: 10px;
|
||||
border-radius: 8px;
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
line-height: 18px;
|
||||
text-align: center;
|
||||
color: var(--td-text-color-secondary);
|
||||
|
||||
.nav-item.active & {
|
||||
background: color-mix(in srgb, var(--td-brand-color) 18%, transparent);
|
||||
color: var(--td-brand-color);
|
||||
}
|
||||
font-size: 11px;
|
||||
line-height: 16px;
|
||||
font-weight: 500;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.section--prompts {
|
||||
|
||||
@@ -1565,20 +1565,15 @@ watch(
|
||||
|
||||
.nav-badge {
|
||||
flex-shrink: 0;
|
||||
min-width: 18px;
|
||||
margin-left: 2px;
|
||||
padding: 0 6px;
|
||||
border-radius: 10px;
|
||||
border-radius: 8px;
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
line-height: 18px;
|
||||
text-align: center;
|
||||
color: var(--td-text-color-secondary);
|
||||
|
||||
.nav-item.active & {
|
||||
background: color-mix(in srgb, var(--td-brand-color) 18%, transparent);
|
||||
color: var(--td-brand-color);
|
||||
}
|
||||
font-size: 11px;
|
||||
line-height: 16px;
|
||||
font-weight: 500;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.settings-content {
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
} from '@/api/datasource'
|
||||
import SettingDrawer from '@/components/settings/SettingDrawer.vue'
|
||||
import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
|
||||
import { getDatasourceIconUrl } from './datasourceIcons'
|
||||
|
||||
const props = defineProps<{
|
||||
kbId: string
|
||||
@@ -102,10 +103,43 @@ function cancelReplaceCredentials() {
|
||||
replaceCredentialsMode.value = false
|
||||
pendingRemoveCredentials.value = false
|
||||
form.value.config.credentials = {}
|
||||
rssAuthHeaders.value = []
|
||||
testResult.value = credentialsConfigured.value ? 'success' : ''
|
||||
testErrorMsg.value = ''
|
||||
}
|
||||
|
||||
interface CustomHeaderItem {
|
||||
key: string
|
||||
value: string
|
||||
}
|
||||
|
||||
const rssAuthHeaders = ref<CustomHeaderItem[]>([])
|
||||
|
||||
function serializeAuthHeaders(items: CustomHeaderItem[]): string {
|
||||
return items
|
||||
.filter(h => h.key.trim())
|
||||
.map(h => `${h.key.trim()}: ${h.value}`)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
function syncRssAuthHeadersToCredentials() {
|
||||
if (form.value.type !== 'rss') return
|
||||
const serialized = serializeAuthHeaders(rssAuthHeaders.value)
|
||||
if (serialized) {
|
||||
form.value.config.credentials.auth_headers = serialized
|
||||
} else {
|
||||
delete form.value.config.credentials.auth_headers
|
||||
}
|
||||
}
|
||||
|
||||
function addRssAuthHeader() {
|
||||
rssAuthHeaders.value.push({ key: '', value: '' })
|
||||
}
|
||||
|
||||
function removeRssAuthHeader(idx: number) {
|
||||
rssAuthHeaders.value.splice(idx, 1)
|
||||
}
|
||||
|
||||
function needsConnectionTest(): boolean {
|
||||
return !(isEdit.value && credentialsConfigured.value && !replaceCredentialsMode.value)
|
||||
}
|
||||
@@ -289,7 +323,16 @@ interface ConnectorDef {
|
||||
permissionDocUrl: string
|
||||
permissionPageUrl: string
|
||||
requiredPermissions: string[]
|
||||
fields: { key: string; labelKey: string; placeholder: string; secret?: boolean; optional?: boolean; hintKey?: string }[]
|
||||
fields: {
|
||||
key: string
|
||||
labelKey: string
|
||||
placeholder: string
|
||||
secret?: boolean
|
||||
optional?: boolean
|
||||
hintKey?: string
|
||||
multiline?: boolean
|
||||
fieldType?: 'custom_headers'
|
||||
}[]
|
||||
}
|
||||
|
||||
const connectorDefs = computed<ConnectorDef[]>(() => [
|
||||
@@ -336,6 +379,18 @@ const connectorDefs = computed<ConnectorDef[]>(() => [
|
||||
{ key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://www.yuque.com', optional: true, hintKey: 'datasource.field.baseUrlHint' },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'rss',
|
||||
available: true,
|
||||
docUrl: '',
|
||||
permissionDocUrl: '',
|
||||
permissionPageUrl: '',
|
||||
requiredPermissions: [],
|
||||
fields: [
|
||||
{ key: 'feed_urls', labelKey: 'datasource.field.feedUrls', placeholder: 'https://example.com/feed.xml', multiline: true, hintKey: 'datasource.field.feedUrlsHint' },
|
||||
{ key: 'auth_headers', labelKey: 'datasource.field.authHeaders', placeholder: '', optional: true, hintKey: 'datasource.field.authHeadersHint', fieldType: 'custom_headers' },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
|
||||
@@ -366,6 +421,7 @@ watch(visible, async (v) => {
|
||||
loadedChildrenIds.value = new Set()
|
||||
loadingChildrenIds.value = new Set()
|
||||
treeFullyLoaded.value = false
|
||||
rssAuthHeaders.value = []
|
||||
|
||||
if (isEdit.value && props.dataSource) {
|
||||
// Reset edit/replace toggle every open so an aborted replace doesn't
|
||||
@@ -416,19 +472,34 @@ watch(
|
||||
{ deep: true },
|
||||
)
|
||||
|
||||
watch(
|
||||
rssAuthHeaders,
|
||||
() => {
|
||||
syncRssAuthHeadersToCredentials()
|
||||
if (needsConnectionTest()) {
|
||||
testResult.value = ''
|
||||
testErrorMsg.value = ''
|
||||
}
|
||||
},
|
||||
{ deep: true },
|
||||
)
|
||||
|
||||
function selectType(def: ConnectorDef) {
|
||||
if (!def.available) return
|
||||
form.value.type = def.type
|
||||
form.value.name = t(`datasource.connector.${def.type}`)
|
||||
form.value.config.credentials = {}
|
||||
rssAuthHeaders.value = []
|
||||
step.value = 1
|
||||
}
|
||||
|
||||
// --- Test connection (stateless, no DB write) ---
|
||||
async function testConnection() {
|
||||
syncRssAuthHeadersToCredentials()
|
||||
if (!isEdit.value || !credentialsConfigured.value || replaceCredentialsMode.value) {
|
||||
const fields = currentDef.value?.fields || []
|
||||
for (const f of fields) {
|
||||
if (f.optional) continue
|
||||
if (f.optional || f.fieldType === 'custom_headers') continue
|
||||
if (!form.value.config.credentials[f.key]) {
|
||||
MessagePlugin.warning(`${t(f.labelKey)} ${t('datasource.isRequired')}`)
|
||||
return
|
||||
@@ -602,13 +673,14 @@ function toggleResource(id: string) {
|
||||
}
|
||||
|
||||
function validateStep1Fields(): boolean {
|
||||
syncRssAuthHeadersToCredentials()
|
||||
if (isEdit.value && credentialsConfigured.value && !replaceCredentialsMode.value) {
|
||||
return true
|
||||
}
|
||||
|
||||
const fields = currentDef.value?.fields || []
|
||||
for (const f of fields) {
|
||||
if (f.optional) continue
|
||||
if (f.optional || f.fieldType === 'custom_headers') continue
|
||||
if (!form.value.config.credentials[f.key]) {
|
||||
MessagePlugin.warning(`${t(f.labelKey)} ${t('datasource.isRequired')}`)
|
||||
return false
|
||||
@@ -657,6 +729,7 @@ function buildConfigPayload(): Record<string, unknown> {
|
||||
// the whole submit on failure so we don't leave the row partially saved.
|
||||
async function commitCredentialsIfNeeded(dsId: string): Promise<boolean> {
|
||||
if (!isEdit.value || !replaceCredentialsMode.value) return true
|
||||
syncRssAuthHeadersToCredentials()
|
||||
const filled = Object.entries(form.value.config.credentials).filter(
|
||||
([, v]) => typeof v === 'string' ? v !== '' : v != null,
|
||||
)
|
||||
@@ -666,6 +739,7 @@ async function commitCredentialsIfNeeded(dsId: string): Promise<boolean> {
|
||||
credentialsConfigured.value = true
|
||||
replaceCredentialsMode.value = false
|
||||
form.value.config.credentials = {}
|
||||
rssAuthHeaders.value = []
|
||||
return true
|
||||
} catch (e: any) {
|
||||
MessagePlugin.error(e?.message || e?.error || t('credential.saveFailed'))
|
||||
@@ -719,6 +793,10 @@ async function handleSubmit() {
|
||||
}
|
||||
|
||||
emit('saved')
|
||||
// Clear before close — otherwise the visible watcher treats the just-saved
|
||||
// row as an abandoned temp draft and DELETEs it (loadResources creates the
|
||||
// row early at step 2 with tempDsId).
|
||||
tempDsId.value = ''
|
||||
visible.value = false
|
||||
} catch (e: any) {
|
||||
MessagePlugin.error(e?.message || e?.error || t('datasource.saveFailed'))
|
||||
@@ -822,6 +900,7 @@ const drawerConfirmText = computed(() => {
|
||||
v-model:visible="visible"
|
||||
:title="drawerTitle"
|
||||
:description="drawerDescription"
|
||||
:class="form.type ? `datasource-editor-drawer datasource-editor-drawer--${form.type}` : 'datasource-editor-drawer'"
|
||||
:hide-footer="step === 0"
|
||||
:confirm-text="drawerConfirmText"
|
||||
:confirm-loading="submitting || (step === 1 && testing)"
|
||||
@@ -830,8 +909,12 @@ const drawerConfirmText = computed(() => {
|
||||
@confirm="handleDrawerConfirm"
|
||||
@cancel="handleClose"
|
||||
>
|
||||
<template v-if="form.type" #headerIcon>
|
||||
<DataSourceTypeIcon :type="form.type" :size="20" />
|
||||
<template v-if="form.type && getDatasourceIconUrl(form.type)" #headerIcon>
|
||||
<img
|
||||
:src="getDatasourceIconUrl(form.type)"
|
||||
:alt="form.type"
|
||||
class="datasource-header-icon__img"
|
||||
>
|
||||
</template>
|
||||
|
||||
<template v-if="step === 1" #footer-left>
|
||||
@@ -879,7 +962,10 @@ const drawerConfirmText = computed(() => {
|
||||
:key="i"
|
||||
:class="['ds-step', { active: step === i, done: step > i }]"
|
||||
>
|
||||
<span class="ds-step-num">{{ step > i ? '✓' : i + 1 }}</span>
|
||||
<span class="ds-step-num">
|
||||
<t-icon v-if="step > i" name="check" class="ds-step-check" />
|
||||
<template v-else>{{ i + 1 }}</template>
|
||||
</span>
|
||||
<span class="ds-step-title">{{ title }}</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -907,10 +993,10 @@ const drawerConfirmText = computed(() => {
|
||||
</section>
|
||||
|
||||
<!-- Step 1: Credentials -->
|
||||
<section v-if="step === 1" class="setting-drawer__section">
|
||||
<template v-if="step === 1">
|
||||
<div
|
||||
v-if="currentDef && currentDef.requiredPermissions.length > 0"
|
||||
class="ds-setup-guide"
|
||||
class="ds-setup-guide ds-setup-guide--standalone"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
@@ -970,29 +1056,33 @@ const drawerConfirmText = computed(() => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label class="form-label">{{ t('datasource.nameLabel') }}</label>
|
||||
<t-input v-model="form.name" :placeholder="t('datasource.namePlaceholder')" />
|
||||
</div>
|
||||
<section class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">{{ t('datasource.sectionBasic') }}</h4>
|
||||
|
||||
<div v-if="currentDef?.docUrl" class="inline-alert">
|
||||
<t-icon name="info-circle-filled" class="inline-alert__icon" />
|
||||
<span class="inline-alert__text">{{ t('datasource.docHint') }}</span>
|
||||
<a
|
||||
:href="currentDef.docUrl"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
class="inline-alert__action doc-link"
|
||||
>
|
||||
{{ t('datasource.openDoc') }}
|
||||
<t-icon name="link" class="link-icon" />
|
||||
</a>
|
||||
</div>
|
||||
<div v-if="currentDef?.docUrl" class="inline-alert">
|
||||
<t-icon name="info-circle-filled" class="inline-alert__icon" />
|
||||
<span class="inline-alert__text">{{ t('datasource.docHint') }}</span>
|
||||
<a
|
||||
:href="currentDef.docUrl"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
class="inline-alert__action doc-link"
|
||||
>
|
||||
{{ t('datasource.openDoc') }}
|
||||
<t-icon name="link" class="link-icon" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="form-item">
|
||||
<label class="form-label">{{ t('datasource.credentialsLabel') }}</label>
|
||||
<div class="form-item">
|
||||
<label class="form-label required">{{ t('datasource.nameLabel') }}</label>
|
||||
<t-input v-model="form.name" :placeholder="t('datasource.namePlaceholder')" />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<template v-if="isEdit && credentialsConfigured && !replaceCredentialsMode">
|
||||
<section class="setting-drawer__section">
|
||||
<h4 class="setting-drawer__section-title">{{ t('datasource.credentialsLabel') }}</h4>
|
||||
|
||||
<div v-if="isEdit && credentialsConfigured && !replaceCredentialsMode" class="form-item">
|
||||
<div
|
||||
class="credential-faux-input"
|
||||
:class="{ 'is-confirm-remove': pendingRemoveCredentials }"
|
||||
@@ -1031,54 +1121,103 @@ const drawerConfirmText = computed(() => {
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-else-if="isEdit && !credentialsConfigured && !replaceCredentialsMode"
|
||||
class="credential-faux-input is-empty"
|
||||
@click="enterReplaceCredentials"
|
||||
class="form-item"
|
||||
>
|
||||
<t-icon name="lock-on" class="credential-status-icon muted" />
|
||||
<span class="credential-faux-text muted">{{ t('credential.unconfigured') }}</span>
|
||||
<div class="credential-actions">
|
||||
<t-button size="small" variant="text" theme="primary" @click.stop="enterReplaceCredentials">
|
||||
{{ t('credential.configure') }}
|
||||
</t-button>
|
||||
<div
|
||||
class="credential-faux-input is-empty"
|
||||
@click="enterReplaceCredentials"
|
||||
>
|
||||
<t-icon name="lock-on" class="credential-status-icon muted" />
|
||||
<span class="credential-faux-text muted">{{ t('credential.unconfigured') }}</span>
|
||||
<div class="credential-actions">
|
||||
<t-button size="small" variant="text" theme="primary" @click.stop="enterReplaceCredentials">
|
||||
{{ t('credential.configure') }}
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="credentialsInputVisible" class="credential-edit">
|
||||
<template v-else-if="credentialsInputVisible">
|
||||
<div
|
||||
v-for="field in currentDef?.fields || []"
|
||||
:key="field.key"
|
||||
class="credential-field"
|
||||
class="form-item"
|
||||
>
|
||||
<label
|
||||
v-if="(currentDef?.fields?.length || 0) > 1"
|
||||
class="credential-field-label"
|
||||
>
|
||||
{{ t(field.labelKey) }}
|
||||
<span v-if="!field.optional" class="required-mark">*</span>
|
||||
</label>
|
||||
<t-input
|
||||
v-model="form.config.credentials[field.key]"
|
||||
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
|
||||
:type="field.secret ? 'password' : 'text'"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
>
|
||||
<template v-if="field.secret" #prefix-icon><t-icon name="lock-on" /></template>
|
||||
</t-input>
|
||||
<p v-if="field.hintKey" class="form-desc">{{ t(field.hintKey) }}</p>
|
||||
<template v-if="field.fieldType === 'custom_headers'">
|
||||
<div class="custom-headers-header">
|
||||
<label class="form-label" style="margin-bottom: 0;">{{ t(field.labelKey) }}</label>
|
||||
<t-button variant="text" size="small" theme="primary" @click="addRssAuthHeader">
|
||||
<template #icon><t-icon name="add" /></template>
|
||||
{{ t('model.editor.customHeadersAdd') }}
|
||||
</t-button>
|
||||
</div>
|
||||
<p v-if="field.hintKey" class="form-desc custom-headers-desc">{{ t(field.hintKey) }}</p>
|
||||
<div v-if="rssAuthHeaders.length > 0" class="custom-headers-list">
|
||||
<div v-for="(item, idx) in rssAuthHeaders" :key="idx" class="custom-header-row">
|
||||
<t-input
|
||||
v-model="item.key"
|
||||
:placeholder="t('model.editor.customHeadersKeyPlaceholder')"
|
||||
class="custom-header-key"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
/>
|
||||
<t-input
|
||||
v-model="item.value"
|
||||
:placeholder="t('model.editor.customHeadersValuePlaceholder')"
|
||||
class="custom-header-value"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
/>
|
||||
<t-button
|
||||
variant="text"
|
||||
shape="square"
|
||||
size="small"
|
||||
class="custom-header-remove"
|
||||
:aria-label="t('common.delete')"
|
||||
@click="removeRssAuthHeader(idx)"
|
||||
>
|
||||
<t-icon name="close" />
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else>
|
||||
<label class="form-label" :class="{ required: !field.optional }">
|
||||
{{ t(field.labelKey) }}
|
||||
</label>
|
||||
<t-textarea
|
||||
v-if="field.multiline"
|
||||
v-model="form.config.credentials[field.key]"
|
||||
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
|
||||
:autosize="{ minRows: 2, maxRows: 6 }"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
/>
|
||||
<t-input
|
||||
v-else
|
||||
v-model="form.config.credentials[field.key]"
|
||||
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
|
||||
:type="field.secret ? 'password' : 'text'"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
>
|
||||
<template v-if="field.secret" #prefix-icon><t-icon name="lock-on" /></template>
|
||||
</t-input>
|
||||
<p v-if="field.hintKey" class="form-desc">{{ t(field.hintKey) }}</p>
|
||||
</template>
|
||||
</div>
|
||||
<div v-if="isEdit && replaceCredentialsMode" class="credential-edit-actions">
|
||||
<t-button size="small" variant="text" @click="cancelReplaceCredentials">
|
||||
{{ t('common.cancel') }}
|
||||
</t-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<!-- Step 2: Select resources -->
|
||||
<section v-if="step === 2" class="setting-drawer__section ds-resource-section">
|
||||
@@ -1280,7 +1419,7 @@ const drawerConfirmText = computed(() => {
|
||||
@import './datasource-surface.less';
|
||||
.ds-steps {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
gap: 8px;
|
||||
margin-bottom: 20px;
|
||||
border-bottom: 1px solid var(--td-component-stroke);
|
||||
padding-bottom: 14px;
|
||||
@@ -1289,22 +1428,32 @@ const drawerConfirmText = computed(() => {
|
||||
.ds-step {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
gap: 8px;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
font-size: 13px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
}
|
||||
|
||||
.ds-step-title {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.ds-step.active {
|
||||
color: var(--td-brand-color);
|
||||
font-weight: 600;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.ds-step.done {
|
||||
color: var(--td-text-color-secondary);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.ds-step-num {
|
||||
flex-shrink: 0;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
border-radius: 50%;
|
||||
@@ -1312,7 +1461,10 @@ const drawerConfirmText = computed(() => {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 12px;
|
||||
border: 1px solid currentColor;
|
||||
font-weight: 600;
|
||||
border: 1px solid var(--td-component-stroke);
|
||||
color: var(--td-text-color-placeholder);
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.ds-step.active .ds-step-num {
|
||||
@@ -1322,9 +1474,13 @@ const drawerConfirmText = computed(() => {
|
||||
}
|
||||
|
||||
.ds-step.done .ds-step-num {
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
color: var(--td-success-color);
|
||||
border-color: color-mix(in srgb, var(--td-success-color) 35%, var(--td-component-stroke));
|
||||
background: color-mix(in srgb, var(--td-brand-color) 12%, transparent);
|
||||
color: var(--td-brand-color);
|
||||
border-color: transparent;
|
||||
}
|
||||
|
||||
.ds-step-check {
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.ds-loading-center {
|
||||
@@ -1422,6 +1578,10 @@ const drawerConfirmText = computed(() => {
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.ds-setup-guide--standalone {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.ds-setup-guide__toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -1594,25 +1754,6 @@ const drawerConfirmText = computed(() => {
|
||||
margin: 0 2px;
|
||||
}
|
||||
|
||||
.credential-edit {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.credential-field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.credential-field-label {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-primary);
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.credential-edit-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
@@ -1625,7 +1766,7 @@ const drawerConfirmText = computed(() => {
|
||||
}
|
||||
|
||||
.form-item {
|
||||
margin-bottom: 16px;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.form-item--flat {
|
||||
@@ -1644,11 +1785,15 @@ const drawerConfirmText = computed(() => {
|
||||
font-weight: 500;
|
||||
margin-bottom: 6px;
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
line-height: 1.4;
|
||||
|
||||
.required-mark {
|
||||
color: var(--td-error-color);
|
||||
margin-left: 2px;
|
||||
&.required::before {
|
||||
content: '*';
|
||||
color: var(--td-error-color);
|
||||
margin-right: 4px;
|
||||
font-weight: 500;
|
||||
line-height: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.form-desc {
|
||||
@@ -1936,6 +2081,55 @@ const drawerConfirmText = computed(() => {
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.custom-headers-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.custom-headers-desc {
|
||||
margin: 0 0 10px 0;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-placeholder);
|
||||
}
|
||||
|
||||
.custom-headers-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.custom-header-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
|
||||
.custom-header-key {
|
||||
flex: 0 0 38%;
|
||||
}
|
||||
|
||||
.custom-header-value {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.custom-header-remove {
|
||||
flex-shrink: 0;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
color: var(--td-text-color-placeholder);
|
||||
border-radius: 6px;
|
||||
transition: all 0.18s ease;
|
||||
|
||||
&:hover {
|
||||
background: var(--td-error-color-light);
|
||||
color: var(--td-error-color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.ds-empty-retry {
|
||||
padding: 0;
|
||||
border: none;
|
||||
@@ -2001,3 +2195,20 @@ const drawerConfirmText = computed(() => {
|
||||
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.05);
|
||||
}
|
||||
</style>
|
||||
|
||||
<!--
|
||||
Drawer header logo — same white badge as list cards / StorageEngineSettings.
|
||||
-->
|
||||
<style lang="less">
|
||||
.datasource-editor-drawer .setting-drawer__header-icon:has(.datasource-header-icon__img) {
|
||||
background: var(--td-bg-color-container, #fff);
|
||||
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
|
||||
}
|
||||
|
||||
.datasource-header-icon__img {
|
||||
display: block;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
object-fit: contain;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -202,11 +202,11 @@ onBeforeUnmount(stopPolling)
|
||||
v-for="ds in dataSources"
|
||||
:key="ds.id"
|
||||
:type="canManageDataSource ? 'button' : undefined"
|
||||
:class="['ds-card', { 'ds-card--clickable': canManageDataSource }]"
|
||||
:class="['ds-card', `ds-card--${ds.type}`, { 'ds-card--clickable': canManageDataSource }]"
|
||||
@click="canManageDataSource ? openEdit(ds) : undefined"
|
||||
>
|
||||
<div class="ds-card__badge">
|
||||
<DataSourceTypeIcon :type="ds.type" :size="22" />
|
||||
<DataSourceTypeIcon :type="ds.type" variant="badge" />
|
||||
</div>
|
||||
<div class="ds-card__body">
|
||||
<div class="ds-card__header">
|
||||
@@ -452,10 +452,23 @@ onBeforeUnmount(stopPolling)
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: color-mix(in srgb, var(--td-brand-color) 10%, transparent);
|
||||
margin-top: 1px;
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
background: rgba(7, 192, 95, 0.12);
|
||||
color: #07c05f;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
&--feishu .ds-card__badge,
|
||||
&--notion .ds-card__badge,
|
||||
&--yuque .ds-card__badge,
|
||||
&--rss .ds-card__badge {
|
||||
background: var(--td-bg-color-container, #fff);
|
||||
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
|
||||
}
|
||||
|
||||
&__body {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
|
||||
@@ -1,20 +1,17 @@
|
||||
<script setup lang="ts">
|
||||
import feishuIcon from '@/assets/img/datasource-feishu.ico'
|
||||
import notionIcon from '@/assets/img/datasource-notion.ico'
|
||||
import yuqueIcon from '@/assets/img/datasource-yuque.ico'
|
||||
import { getDatasourceIconUrl, datasourceIconMap } from './datasourceIcons'
|
||||
|
||||
const props = withDefaults(defineProps<{
|
||||
type: string
|
||||
size?: number
|
||||
/** inline: 类型选择等小尺寸场景;badge: 嵌入 ds-card__badge 等父级徽章容器 */
|
||||
variant?: 'inline' | 'badge'
|
||||
}>(), {
|
||||
size: 20,
|
||||
variant: 'inline',
|
||||
})
|
||||
|
||||
const iconMap: Record<string, string> = {
|
||||
feishu: feishuIcon,
|
||||
notion: notionIcon,
|
||||
yuque: yuqueIcon,
|
||||
}
|
||||
const iconMap = datasourceIconMap
|
||||
|
||||
function fallbackText(type: string) {
|
||||
switch (type) {
|
||||
@@ -33,13 +30,15 @@ function fallbackText(type: string) {
|
||||
<template>
|
||||
<span
|
||||
class="ds-type-icon"
|
||||
:style="{ width: `${size}px`, height: `${size}px` }"
|
||||
:class="`ds-type-icon--${variant}`"
|
||||
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
|
||||
>
|
||||
<img
|
||||
v-if="iconMap[type]"
|
||||
:src="iconMap[type]"
|
||||
:alt="type"
|
||||
:style="{ width: `${size}px`, height: `${size}px` }"
|
||||
class="ds-type-icon__img"
|
||||
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
|
||||
>
|
||||
<span v-else class="ds-type-icon-fallback">{{ fallbackText(type) }}</span>
|
||||
</span>
|
||||
@@ -52,18 +51,41 @@ function fallbackText(type: string) {
|
||||
justify-content: center;
|
||||
flex-shrink: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.ds-type-icon--inline {
|
||||
border-radius: 6px;
|
||||
background: var(--td-bg-color-component);
|
||||
}
|
||||
|
||||
.ds-type-icon img {
|
||||
.ds-type-icon--inline .ds-type-icon__img {
|
||||
display: block;
|
||||
object-fit: cover;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.ds-type-icon-fallback {
|
||||
.ds-type-icon--inline .ds-type-icon-fallback {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: var(--td-text-color-placeholder);
|
||||
}
|
||||
|
||||
.ds-type-icon--badge {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.ds-type-icon--badge .ds-type-icon__img {
|
||||
display: block;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.ds-type-icon--badge .ds-type-icon-fallback {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
color: inherit;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import feishuIcon from '@/assets/img/datasource-feishu.ico'
|
||||
import notionIcon from '@/assets/img/datasource-notion.ico'
|
||||
import yuqueIcon from '@/assets/img/datasource-yuque.ico'
|
||||
import rssIcon from '@/assets/img/datasource-rss.svg'
|
||||
|
||||
export const datasourceIconMap: Record<string, string> = {
|
||||
feishu: feishuIcon,
|
||||
notion: notionIcon,
|
||||
yuque: yuqueIcon,
|
||||
rss: rssIcon,
|
||||
}
|
||||
|
||||
export function getDatasourceIconUrl(type: string): string | undefined {
|
||||
return datasourceIconMap[type]
|
||||
}
|
||||
@@ -3,6 +3,7 @@ module github.com/Tencent/WeKnora
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
codeberg.org/readeck/go-readability/v2 v2.1.2
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.1
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
@@ -35,6 +36,7 @@ require (
|
||||
github.com/mark3labs/mcp-go v0.52.0
|
||||
github.com/milvus-io/milvus/client/v2 v2.6.4
|
||||
github.com/minio/minio-go/v7 v7.1.0
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
github.com/neo4j/neo4j-go-driver/v6 v6.0.0
|
||||
github.com/ollama/ollama v0.23.2
|
||||
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
|
||||
@@ -176,6 +178,7 @@ require (
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
@@ -183,6 +186,7 @@ require (
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
github.com/godbus/dbus/v5 v5.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/flatbuffers v25.12.19+incompatible // indirect
|
||||
@@ -194,6 +198,7 @@ require (
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
|
||||
github.com/itlightning/dateparse v0.2.1 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -227,6 +232,7 @@ require (
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
|
||||
@@ -1324,6 +1324,8 @@ cloud.google.com/go/workflows v1.12.1/go.mod h1:5A95OhD/edtOhQd/O741NSfIMezNTbCw
|
||||
cloud.google.com/go/workflows v1.12.2/go.mod h1:+OmBIgNqYJPVggnMo9nqmizW0qEXHhmnAzK/CnBqsHc=
|
||||
cloud.google.com/go/workflows v1.12.3/go.mod h1:fmOUeeqEwPzIU81foMjTRQIdwQHADi/vEr1cx9R1m5g=
|
||||
cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w=
|
||||
codeberg.org/readeck/go-readability/v2 v2.1.2 h1:JBrdyYJBRPMBbodLM1b5KxCSDH+JqCkGcuVRD7ICBAw=
|
||||
codeberg.org/readeck/go-readability/v2 v2.1.2/go.mod h1:Ut31sW4osSrJPR3T8eQslMh4+jbwimXqn0w0ReCT+PU=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
@@ -1773,6 +1775,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
|
||||
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
@@ -1809,6 +1813,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
@@ -2026,6 +2032,8 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
|
||||
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
|
||||
github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
|
||||
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
|
||||
github.com/itlightning/dateparse v0.2.1 h1:AB0NJTyI0HYcerEUMovKZOiQVBg1mBPxgAnWQwzLP6g=
|
||||
github.com/itlightning/dateparse v0.2.1/go.mod h1:xHlmL8lT0L9JIBlaKotRwsoDYpKJskXpiU9ZwbbSkNA=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -2212,6 +2220,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
|
||||
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
|
||||
@@ -57,6 +57,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
feishuConnector "github.com/Tencent/WeKnora/internal/datasource/connector/feishu"
|
||||
notionConnector "github.com/Tencent/WeKnora/internal/datasource/connector/notion"
|
||||
rssConnector "github.com/Tencent/WeKnora/internal/datasource/connector/rss"
|
||||
yuqueConnector "github.com/Tencent/WeKnora/internal/datasource/connector/yuque"
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/handler"
|
||||
@@ -1365,6 +1366,9 @@ func initConnectorRegistry() (*datasource.ConnectorRegistry, error) {
|
||||
if err := registry.Register(yuqueConnector.NewConnector()); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("register yuque connector: %w", err))
|
||||
}
|
||||
if err := registry.Register(rssConnector.NewConnector()); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("register rss connector: %w", err))
|
||||
}
|
||||
|
||||
// Future connectors will be registered here:
|
||||
// if err := registry.Register(confluenceConnector.NewConnector()); err != nil { ... }
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
readability "codeberg.org/readeck/go-readability/v2"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
// requestTimeout bounds a single feed or article fetch.
|
||||
requestTimeout = 20 * time.Second
|
||||
|
||||
// maxFeedSize caps the feed body to avoid memory blowups on hostile feeds.
|
||||
maxFeedSize = 10 * 1024 * 1024 // 10 MB
|
||||
|
||||
// maxArticleSize caps a single article page body.
|
||||
maxArticleSize = 5 * 1024 * 1024 // 5 MB
|
||||
|
||||
// defaultUserAgent is sent on every request; some feeds reject empty UAs.
|
||||
defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
|
||||
)
|
||||
|
||||
// client performs SSRF-safe HTTP fetches with optional custom headers.
|
||||
type client struct {
|
||||
httpClient *http.Client
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
func newClient(headers map[string]string) *client {
|
||||
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
cfg.Timeout = requestTimeout
|
||||
return &client{
|
||||
httpClient: utils.NewSSRFSafeHTTPClient(cfg),
|
||||
headers: headers,
|
||||
}
|
||||
}
|
||||
|
||||
// fetch retrieves rawURL with SSRF validation and size limiting. The user's
|
||||
// custom headers are applied first, then a default User-Agent / Accept if the
|
||||
// user did not override them.
|
||||
func (c *client) fetch(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {
|
||||
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
|
||||
return nil, fmt.Errorf("URL rejected: %w", err)
|
||||
}
|
||||
if _, err := url.Parse(rawURL); err != nil {
|
||||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for k, v := range c.headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
if req.Header.Get("Accept") == "" {
|
||||
req.Header.Set("Accept",
|
||||
"application/rss+xml, application/atom+xml, application/xml, text/xml, application/json, text/html;q=0.9, */*;q=0.8")
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body failed: %w", err)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// fetchFeed retrieves the raw bytes of a feed document.
|
||||
func (c *client) fetchFeed(ctx context.Context, feedURL string) ([]byte, error) {
|
||||
return c.fetch(ctx, feedURL, maxFeedSize)
|
||||
}
|
||||
|
||||
// extractArticle fetches an article page and returns the readability-cleaned
|
||||
// main content as HTML, plus the extracted title (may be empty). Returns an
|
||||
// error if the page can't be fetched or no readable content is found, so the
|
||||
// caller can fall back to feed-provided content.
|
||||
func (c *client) extractArticle(ctx context.Context, articleURL string) (contentHTML, title string, err error) {
|
||||
body, err := c.fetch(ctx, articleURL, maxArticleSize)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
pageURL, _ := url.Parse(articleURL)
|
||||
article, err := readability.FromReader(bytes.NewReader(body), pageURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("readability parse: %w", err)
|
||||
}
|
||||
if article.Node == nil {
|
||||
return "", "", fmt.Errorf("no readable content extracted")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := article.RenderHTML(&buf); err != nil {
|
||||
return "", "", fmt.Errorf("render article html: %w", err)
|
||||
}
|
||||
return buf.String(), article.Title(), nil
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
// Compile-time proof that *Connector satisfies the datasource.Connector interface.
|
||||
var _ datasource.Connector = (*Connector)(nil)
|
||||
|
||||
// Connector implements datasource.Connector for RSS/Atom/JSON feeds.
|
||||
type Connector struct{}
|
||||
|
||||
// NewConnector creates a new RSS connector.
|
||||
func NewConnector() *Connector { return &Connector{} }
|
||||
|
||||
// Type returns the connector type identifier.
|
||||
func (c *Connector) Type() string { return types.ConnectorTypeRSS }
|
||||
|
||||
// Validate verifies that every configured feed URL is reachable and parses as
|
||||
// a valid feed.
|
||||
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
for _, feedURL := range cfg.feedURLList() {
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch feed %s: %w", feedURL, err)
|
||||
}
|
||||
if _, err := parser.Parse(bytes.NewReader(data)); err != nil {
|
||||
return fmt.Errorf("parse feed %s: %w", feedURL, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolveResourceAncestors has nothing to do: feeds are a flat list with no
|
||||
// nesting, so a selection has no ancestors to reveal.
|
||||
func (c *Connector) ResolveResourceAncestors(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]string, error) {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// ListResources returns one resource per configured feed URL. The feed is
|
||||
// fetched so the resource can carry its real title; a feed that fails to fetch
|
||||
// still appears (named by URL) with an error note, so the user can deselect it
|
||||
// instead of the whole listing failing.
|
||||
func (c *Connector) ListResources(
|
||||
ctx context.Context, config *types.DataSourceConfig, parentID string,
|
||||
) ([]types.Resource, error) {
|
||||
// Feeds are flat: a lazy-load request for a specific parent has nothing extra.
|
||||
if parentID != "" {
|
||||
return []types.Resource{}, nil
|
||||
}
|
||||
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
feedURLs := cfg.feedURLList()
|
||||
out := make([]types.Resource, 0, len(feedURLs))
|
||||
for _, feedURL := range feedURLs {
|
||||
res := types.Resource{
|
||||
ExternalID: feedURL,
|
||||
Type: "feed",
|
||||
Name: feedURL,
|
||||
URL: feedURL,
|
||||
}
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[RSS] list: fetch %s failed: %v", feedURL, err)
|
||||
res.Description = "fetch failed: " + err.Error()
|
||||
out = append(out, res)
|
||||
continue
|
||||
}
|
||||
feed, err := parser.Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[RSS] list: parse %s failed: %v", feedURL, err)
|
||||
res.Description = "parse failed: " + err.Error()
|
||||
out = append(out, res)
|
||||
continue
|
||||
}
|
||||
if title := strings.TrimSpace(feed.Title); title != "" {
|
||||
res.Name = title
|
||||
}
|
||||
res.Description = strings.TrimSpace(feed.Description)
|
||||
if feed.Link != "" {
|
||||
res.URL = feed.Link
|
||||
}
|
||||
if feed.UpdatedParsed != nil {
|
||||
res.ModifiedAt = *feed.UpdatedParsed
|
||||
}
|
||||
res.Metadata = map[string]interface{}{"item_count": len(feed.Items)}
|
||||
out = append(out, res)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FetchAll performs a full sync of the specified feeds (or all configured feeds
|
||||
// when resourceIDs is empty).
|
||||
func (c *Connector) FetchAll(
|
||||
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
|
||||
) ([]types.FetchedItem, error) {
|
||||
items, _, err := c.walk(ctx, config, resourceIDs, nil, false)
|
||||
return items, err
|
||||
}
|
||||
|
||||
// FetchIncremental returns only items whose fingerprint changed since the prior
|
||||
// cursor. Deletions are intentionally not emitted (feeds drop old items as a
|
||||
// matter of course).
|
||||
func (c *Connector) FetchIncremental(
|
||||
ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor,
|
||||
) ([]types.FetchedItem, *types.SyncCursor, error) {
|
||||
var prev *rssCursor
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
var p rssCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &p)
|
||||
prev = &p
|
||||
}
|
||||
|
||||
items, newCursor, err := c.walk(ctx, config, config.ResourceIDs, prev, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
cursorMap := make(map[string]interface{})
|
||||
b, _ := json.Marshal(newCursor)
|
||||
_ = json.Unmarshal(b, &cursorMap)
|
||||
|
||||
return items, &types.SyncCursor{
|
||||
LastSyncTime: newCursor.LastSyncTime,
|
||||
ConnectorCursor: cursorMap,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// walk is the shared implementation for FetchAll / FetchIncremental.
|
||||
//
|
||||
// When incremental is true, items whose fingerprint matches the prior cursor
|
||||
// are skipped (no article re-fetch). The returned cursor always reflects the
|
||||
// full current item set so a later sync can detect changes.
|
||||
func (c *Connector) walk(
|
||||
ctx context.Context,
|
||||
config *types.DataSourceConfig,
|
||||
resourceIDs []string,
|
||||
prev *rssCursor,
|
||||
incremental bool,
|
||||
) ([]types.FetchedItem, *rssCursor, error) {
|
||||
cfg, err := parseConfig(config)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Default to all configured feeds when no explicit selection was made.
|
||||
feedURLs := resourceIDs
|
||||
if len(feedURLs) == 0 {
|
||||
feedURLs = cfg.feedURLList()
|
||||
}
|
||||
|
||||
cli := newClient(cfg.parseHeaders())
|
||||
parser := gofeed.NewParser()
|
||||
|
||||
newCursor := &rssCursor{
|
||||
LastSyncTime: time.Now().UTC(),
|
||||
FeedItems: make(map[string]map[string]string),
|
||||
}
|
||||
var out []types.FetchedItem
|
||||
|
||||
for _, feedURL := range feedURLs {
|
||||
data, err := cli.fetchFeed(ctx, feedURL)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("fetch feed %s: %w", feedURL, err)
|
||||
}
|
||||
feed, err := parser.Parse(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse feed %s: %w", feedURL, err)
|
||||
}
|
||||
|
||||
newCursor.FeedItems[feedURL] = make(map[string]string)
|
||||
var prevItems map[string]string
|
||||
if incremental && prev != nil {
|
||||
prevItems = prev.FeedItems[feedURL]
|
||||
}
|
||||
|
||||
var kept, skipped int
|
||||
for _, item := range feed.Items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
|
||||
if itemID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
feedContent := firstNonEmpty(item.Content, item.Description)
|
||||
fp := itemFingerprint(item.UpdatedParsed, item.PublishedParsed, feedContent)
|
||||
newCursor.FeedItems[feedURL][itemID] = fp
|
||||
|
||||
if incremental && prevItems != nil && prevItems[itemID] == fp {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
|
||||
out = append(out, c.buildItem(ctx, cli, feed, item, feedURL, itemID, feedContent))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[RSS] feed %s: items=%d fetched=%d skipped=%d",
|
||||
feedURL, len(feed.Items), kept, skipped)
|
||||
}
|
||||
|
||||
if !incremental {
|
||||
return out, newCursor, nil
|
||||
}
|
||||
return out, newCursor, nil
|
||||
}
|
||||
|
||||
// buildItem assembles a FetchedItem for a single feed entry, resolving the
|
||||
// best available content (full-text article > feed content) and converting it
|
||||
// to Markdown.
|
||||
func (c *Connector) buildItem(
|
||||
ctx context.Context,
|
||||
cli *client,
|
||||
feed *gofeed.Feed,
|
||||
item *gofeed.Item,
|
||||
feedURL, itemID, feedContent string,
|
||||
) types.FetchedItem {
|
||||
title := firstNonEmpty(item.Title, "untitled")
|
||||
|
||||
// Prefer full article text; fall back to feed-provided content on failure.
|
||||
contentHTML := feedContent
|
||||
if strings.TrimSpace(item.Link) != "" {
|
||||
if articleHTML, articleTitle, err := cli.extractArticle(ctx, item.Link); err == nil {
|
||||
contentHTML = articleHTML
|
||||
if item.Title == "" && articleTitle != "" {
|
||||
title = articleTitle
|
||||
}
|
||||
} else {
|
||||
logger.Warnf(ctx, "[RSS] full-text fetch failed for %s (using feed content): %v", item.Link, err)
|
||||
}
|
||||
}
|
||||
|
||||
content := htmlToMarkdown(contentHTML)
|
||||
|
||||
updatedAt := time.Now().UTC()
|
||||
switch {
|
||||
case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
|
||||
updatedAt = *item.UpdatedParsed
|
||||
case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
|
||||
updatedAt = *item.PublishedParsed
|
||||
}
|
||||
|
||||
author := ""
|
||||
if item.Author != nil {
|
||||
author = item.Author.Name
|
||||
}
|
||||
|
||||
return types.FetchedItem{
|
||||
ExternalID: itemID,
|
||||
Title: title,
|
||||
Content: []byte(content),
|
||||
ContentType: "text/markdown",
|
||||
FileName: sanitizeFileName(title) + ".md",
|
||||
URL: item.Link,
|
||||
UpdatedAt: updatedAt,
|
||||
SourceResourceID: feedURL,
|
||||
Metadata: map[string]string{
|
||||
"channel": types.ChannelRSS,
|
||||
"feed_url": feedURL,
|
||||
"feed_title": feed.Title,
|
||||
"guid": item.GUID,
|
||||
"link": item.Link,
|
||||
"author": author,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// htmlToMarkdown converts HTML to Markdown, returning the trimmed original on
|
||||
// conversion failure so we never silently drop content.
|
||||
func htmlToMarkdown(html string) string {
|
||||
if strings.TrimSpace(html) == "" {
|
||||
return ""
|
||||
}
|
||||
md, err := htmltomd.ConvertString(html)
|
||||
if err != nil || strings.TrimSpace(md) == "" {
|
||||
return strings.TrimSpace(html)
|
||||
}
|
||||
return strings.TrimSpace(md)
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1)
|
||||
// are reachable. Production keeps the default strict SSRF policy.
|
||||
func TestMain(m *testing.M) {
|
||||
_ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,::1")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
const longArticleBody = `<p>This is the first paragraph of a reasonably long article that the ` +
|
||||
`readability extractor should detect as the main content of the page. It contains ` +
|
||||
`enough words to clear the minimum content threshold used by the algorithm.</p>` +
|
||||
`<p>The second paragraph continues the discussion with more sentences so that the ` +
|
||||
`scoring heuristics confidently select this block over navigation and footer noise.</p>` +
|
||||
`<p>A third paragraph adds further substance, ensuring the article is unmistakably ` +
|
||||
`the dominant readable region of the document under test.</p>`
|
||||
|
||||
// fakeFeed spins up an httptest server serving an RSS feed and article pages.
|
||||
type fakeFeed struct {
|
||||
server *httptest.Server
|
||||
feedTitle string
|
||||
itemContent string // optional <description>/content for items
|
||||
}
|
||||
|
||||
func newFakeFeed(t *testing.T) *fakeFeed {
|
||||
t.Helper()
|
||||
f := &fakeFeed{feedTitle: "Test Feed"}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/article/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, `<!DOCTYPE html><html><head><title>%s</title></head>`+
|
||||
`<body><nav>menu</nav><article><h1>Heading</h1>%s</article><footer>foot</footer></body></html>`,
|
||||
"Full "+r.URL.Path, longArticleBody)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/feed.xml", func(w http.ResponseWriter, r *http.Request) {
|
||||
if auth := r.Header.Get("X-Test-Auth"); f.itemContent == "needs-auth" && auth != "secret" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/rss+xml")
|
||||
base := "http://" + r.Host
|
||||
desc := "summary fallback"
|
||||
fmt.Fprintf(w, `<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>%s</title>
|
||||
<link>%s</link>
|
||||
<description>A test feed</description>
|
||||
<item>
|
||||
<title>Article One</title>
|
||||
<link>%s/article/a1</link>
|
||||
<guid>guid-1</guid>
|
||||
<pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate>
|
||||
<description>%s</description>
|
||||
</item>
|
||||
<item>
|
||||
<title>Article Two</title>
|
||||
<link>%s/article/a2</link>
|
||||
<guid>guid-2</guid>
|
||||
<pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
|
||||
<description>%s</description>
|
||||
</item>
|
||||
</channel></rss>`, f.feedTitle, base, base, desc, base, desc)
|
||||
})
|
||||
|
||||
f.server = httptest.NewServer(mux)
|
||||
t.Cleanup(f.server.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeFeed) feedURL() string { return f.server.URL + "/feed.xml" }
|
||||
|
||||
func makeConfig(feedURLs string, headers string) *types.DataSourceConfig {
|
||||
creds := map[string]interface{}{"feed_urls": feedURLs}
|
||||
if headers != "" {
|
||||
creds["auth_headers"] = headers
|
||||
}
|
||||
return &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeRSS,
|
||||
Credentials: creds,
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Type(t *testing.T) {
|
||||
if NewConnector().Type() != types.ConnectorTypeRSS {
|
||||
t.Fatalf("Type() = %q, want %q", NewConnector().Type(), types.ConnectorTypeRSS)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_RequiresFeedURLs(t *testing.T) {
|
||||
if _, err := parseConfig(makeConfig(" ", "")); err == nil {
|
||||
t.Fatal("expected error when feed_urls is blank")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_FeedURLList_SplitsAndDedupes(t *testing.T) {
|
||||
cfg := &Config{FeedURLs: "https://a.com/f, https://b.com/f\nhttps://a.com/f\n"}
|
||||
got := cfg.feedURLList()
|
||||
want := []string{"https://a.com/f", "https://b.com/f"}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Fatalf("feedURLList = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ParseHeaders(t *testing.T) {
|
||||
cfg := &Config{AuthHeaders: "Authorization: Bearer x\nX-Foo: bar \nbroken-line\n: noname"}
|
||||
got := cfg.parseHeaders()
|
||||
if got["Authorization"] != "Bearer x" {
|
||||
t.Errorf("Authorization = %q", got["Authorization"])
|
||||
}
|
||||
if got["X-Foo"] != "bar" {
|
||||
t.Errorf("X-Foo = %q", got["X-Foo"])
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Errorf("expected 2 headers, got %d: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Validate_Success(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_Validate_PrivateFeedWithHeader(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
f.itemContent = "needs-auth"
|
||||
|
||||
// Without the header → 401.
|
||||
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err == nil {
|
||||
t.Fatal("expected error without auth header")
|
||||
}
|
||||
// With the header → success.
|
||||
if err := NewConnector().Validate(
|
||||
context.Background(), makeConfig(f.feedURL(), "X-Test-Auth: secret"),
|
||||
); err != nil {
|
||||
t.Fatalf("Validate with header error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_ListResources(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
res, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources error: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 resource, got %d", len(res))
|
||||
}
|
||||
if res[0].ExternalID != f.feedURL() {
|
||||
t.Errorf("ExternalID = %q, want %q", res[0].ExternalID, f.feedURL())
|
||||
}
|
||||
if res[0].Name != "Test Feed" {
|
||||
t.Errorf("Name = %q, want %q", res[0].Name, "Test Feed")
|
||||
}
|
||||
|
||||
// Non-empty parentID → empty (feeds are flat).
|
||||
children, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "feed-x")
|
||||
if err != nil {
|
||||
t.Fatalf("ListResources(parent) error: %v", err)
|
||||
}
|
||||
if len(children) != 0 {
|
||||
t.Fatalf("expected no children, got %d", len(children))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_FetchAll_FullTextMarkdown(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
items, err := NewConnector().FetchAll(context.Background(), makeConfig(f.feedURL(), ""), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAll error: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected 2 items, got %d", len(items))
|
||||
}
|
||||
|
||||
it := items[0]
|
||||
if it.ExternalID != "guid-1" {
|
||||
t.Errorf("ExternalID = %q, want guid-1", it.ExternalID)
|
||||
}
|
||||
if it.ContentType != "text/markdown" {
|
||||
t.Errorf("ContentType = %q, want text/markdown", it.ContentType)
|
||||
}
|
||||
if it.Metadata["channel"] != types.ChannelRSS {
|
||||
t.Errorf("channel = %q, want %q", it.Metadata["channel"], types.ChannelRSS)
|
||||
}
|
||||
// Full text from the article page should be present (not the short summary).
|
||||
if !strings.Contains(string(it.Content), "first paragraph") {
|
||||
t.Errorf("expected full article text in content, got: %q", string(it.Content))
|
||||
}
|
||||
if !strings.HasSuffix(it.FileName, ".md") {
|
||||
t.Errorf("FileName = %q, want .md suffix", it.FileName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_FetchIncremental_SkipsUnchanged(t *testing.T) {
|
||||
f := newFakeFeed(t)
|
||||
cfg := makeConfig(f.feedURL(), "")
|
||||
cfg.ResourceIDs = []string{f.feedURL()}
|
||||
|
||||
// First sync: everything is new.
|
||||
items, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first FetchIncremental error: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("expected 2 items on first sync, got %d", len(items))
|
||||
}
|
||||
if cursor == nil {
|
||||
t.Fatal("expected non-nil cursor")
|
||||
}
|
||||
|
||||
// Second sync with the returned cursor: nothing changed → no items.
|
||||
items2, _, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
|
||||
if err != nil {
|
||||
t.Fatalf("second FetchIncremental error: %v", err)
|
||||
}
|
||||
if len(items2) != 0 {
|
||||
t.Fatalf("expected 0 items on unchanged second sync, got %d", len(items2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
|
||||
got, err := NewConnector().ResolveResourceAncestors(
|
||||
context.Background(), makeConfig("https://a.com/f", ""), []string{"x"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveResourceAncestors error: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("expected empty ancestors, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package rss implements the RSS/Atom data source connector for WeKnora.
|
||||
//
|
||||
// It syncs articles from one or more RSS/Atom/JSON feeds into a WeKnora
|
||||
// knowledge base. Each configured feed URL is treated as a selectable
|
||||
// resource; each feed item becomes a knowledge entry whose body is the
|
||||
// article's full text rendered as Markdown.
|
||||
//
|
||||
// Capabilities:
|
||||
// - Private feeds: optional custom HTTP headers (e.g. "Authorization: Bearer …")
|
||||
// are attached to every request, both to the feed and to article pages.
|
||||
// - Full text: when an item exposes a link, the article page is fetched and
|
||||
// run through a readability extractor; the cleaned HTML is converted to
|
||||
// Markdown. Feed-provided content (content:encoded / description) is used
|
||||
// as a fallback.
|
||||
// - Incremental: a per-feed fingerprint (item updated/published time, or a
|
||||
// content hash when no timestamp is present) detects changed items so only
|
||||
// new/updated articles are re-fetched. Deletions are NOT synced — feeds
|
||||
// routinely drop old items, which would otherwise look like deletions.
|
||||
//
|
||||
// All outbound requests go through the SSRF-safe HTTP client so a malicious
|
||||
// feed cannot redirect WeKnora to internal services.
|
||||
package rss
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// Config holds RSS-specific configuration parsed from DataSourceConfig.Credentials.
|
||||
//
|
||||
// Both fields live under Credentials (rather than Settings) because the
|
||||
// existing data-source editor UI only renders credential inputs, and because
|
||||
// AuthHeaders may carry secrets that must be encrypted at rest. FeedURLs are
|
||||
// not secret but ride along for the same single-input flow.
|
||||
type Config struct {
|
||||
// FeedURLs is a newline- or comma-separated list of feed URLs.
|
||||
FeedURLs string `json:"feed_urls"`
|
||||
|
||||
// AuthHeaders is an optional newline-separated list of custom request
|
||||
// headers in "Name: Value" form, applied to every outbound request.
|
||||
AuthHeaders string `json:"auth_headers,omitempty"`
|
||||
}
|
||||
|
||||
// parseConfig extracts and validates RSS configuration from the credentials map.
|
||||
func parseConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("%w: config is nil", datasource.ErrInvalidConfig)
|
||||
}
|
||||
credBytes, err := json.Marshal(config.Credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal credentials: %w", err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(credBytes, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse rss credentials: %w", err)
|
||||
}
|
||||
if len(cfg.feedURLList()) == 0 {
|
||||
return nil, fmt.Errorf("%w: feed_urls is required", datasource.ErrInvalidCredentials)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// feedURLList splits FeedURLs on newlines and commas, trims, dedupes (order
|
||||
// preserved), and drops blanks.
|
||||
func (c *Config) feedURLList() []string {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
raw := strings.FieldsFunc(c.FeedURLs, func(r rune) bool {
|
||||
return r == '\n' || r == '\r' || r == ','
|
||||
})
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, u := range raw {
|
||||
u = strings.TrimSpace(u)
|
||||
if u == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[u]; ok {
|
||||
continue
|
||||
}
|
||||
seen[u] = struct{}{}
|
||||
out = append(out, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseHeaders turns the newline-separated "Name: Value" AuthHeaders blob into
|
||||
// a map. Lines without a colon, or with an empty name, are skipped.
|
||||
func (c *Config) parseHeaders() map[string]string {
|
||||
if c == nil || strings.TrimSpace(c.AuthHeaders) == "" {
|
||||
return nil
|
||||
}
|
||||
headers := make(map[string]string)
|
||||
for _, line := range strings.Split(c.AuthHeaders, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(line, ":")
|
||||
if idx <= 0 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(line[:idx])
|
||||
value := strings.TrimSpace(line[idx+1:])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
headers[name] = value
|
||||
}
|
||||
if len(headers) == 0 {
|
||||
return nil
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// rssCursor stores incremental sync state.
|
||||
//
|
||||
// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is the
|
||||
// item's updated/published timestamp (RFC3339) or, when no timestamp exists,
|
||||
// a "h:<sha256-prefix>" content hash. An item is considered unchanged when its
|
||||
// fingerprint matches the stored one.
|
||||
type rssCursor struct {
|
||||
LastSyncTime time.Time `json:"last_sync_time"`
|
||||
FeedItems map[string]map[string]string `json:"feed_items,omitempty"`
|
||||
}
|
||||
|
||||
// itemFingerprint returns a change-detection token for a feed item. It prefers
|
||||
// the explicit updated/published timestamp; absent that, it hashes the content
|
||||
// so edits to timestamp-less items are still detected.
|
||||
func itemFingerprint(updated, published *time.Time, content string) string {
|
||||
if updated != nil && !updated.IsZero() {
|
||||
return updated.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if published != nil && !published.IsZero() {
|
||||
return published.UTC().Format(time.RFC3339)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
return "h:" + hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
// firstNonEmpty returns the first non-empty trimmed string among the args.
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters invalid in filenames and truncates to a
|
||||
// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
|
||||
func sanitizeFileName(name string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
"\n", " ", "\r", " ", "\t", " ",
|
||||
)
|
||||
result := strings.TrimSpace(replacer.Replace(name))
|
||||
if result == "" {
|
||||
return "untitled"
|
||||
}
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -53,6 +53,10 @@ func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) UpdateSyncState(ctx context.Context, ds *types.DataSource) error {
|
||||
return r.Update(ctx, ds)
|
||||
}
|
||||
|
||||
func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -117,6 +121,10 @@ func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) UpdateResult(ctx context.Context, log *types.SyncLog) error {
|
||||
return r.Update(ctx, log)
|
||||
}
|
||||
|
||||
func (r *fakeSyncLogRepo) CancelPendingByDataSource(_ context.Context, dsID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ const (
|
||||
ChannelIM = "im" // Generic IM channel
|
||||
ChannelNotion = "notion" // Notion
|
||||
ChannelYuque = "yuque" // Yuque (语雀)
|
||||
ChannelRSS = "rss" // RSS / Atom feed
|
||||
)
|
||||
|
||||
// Knowledge parse status constants
|
||||
|
||||
Reference in New Issue
Block a user