feat(datasource): add RSS feed connector with full-text ingestion

Enable private RSS feeds as a knowledge base data source with custom
headers, article body fetching, and Markdown output for incremental sync.
This commit is contained in:
wizardchen
2026-06-23 14:58:45 +08:00
committed by lyingbug
parent 94ce8fedb1
commit e40ad802d7
21 changed files with 1328 additions and 131 deletions
@@ -0,0 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" role="img" aria-label="RSS">
<circle cx="14" cy="34" r="4" fill="#FF6600"/>
<path d="M10 22a18 18 0 0 1 18 18" fill="none" stroke="#FF6600" stroke-width="4" stroke-linecap="round"/>
<path d="M10 14a26 26 0 0 1 26 26" fill="none" stroke="#FF6600" stroke-width="4" stroke-linecap="round"/>
</svg>

After

Width:  |  Height:  |  Size: 361 B

@@ -7,9 +7,9 @@
<div class="setting-drawer-resize-line" />
</div>
</teleport>
<t-drawer v-model:visible="drawerVisible" :size="effectiveWidth" :z-index="2500" placement="right"
<t-drawer v-model:visible="drawerVisible" v-bind="drawerPassthroughAttrs" :size="effectiveWidth" :z-index="2500" placement="right"
attach="body" destroy-on-close
:class="['setting-drawer', { 'setting-drawer--resizing': drawerResizing }]">
:class="drawerClass">
<!--
Custom header. We replace TDesign's default header so we can put a leading
icon badge and an optional subtitle (description) right next to the title,
@@ -55,7 +55,7 @@
</template>
<script setup lang="ts">
import { computed, ref, onMounted, onUnmounted } from 'vue'
import { ref, computed, useAttrs, onMounted, onUnmounted } from 'vue'
import { useI18n } from 'vue-i18n'
interface Props {
@@ -90,6 +90,8 @@ interface Props {
hideFooter?: boolean
}
defineOptions({ inheritAttrs: false })
const props = withDefaults(defineProps<Props>(), {
description: '',
icon: '',
@@ -112,6 +114,12 @@ const emit = defineEmits<{
}>()
const { t } = useI18n()
const attrs = useAttrs()
const drawerPassthroughAttrs = computed(() => {
const { class: _class, ...rest } = attrs
return rest
})
// ---------- visibility ----------
const drawerVisible = computed({
@@ -172,6 +180,13 @@ const persistWidth = (width: number) => {
// ---------- Custom drag-resize (visible handle, same as doc-content) ----------
const drawerResizing = ref(false)
const drawerClass = computed(() => [
'setting-drawer',
attrs.class,
{ 'setting-drawer--resizing': drawerResizing.value },
])
let resizeStartX = 0
let resizeStartWidth = 0
+7
View File
@@ -5418,6 +5418,7 @@ export default {
editTitle: 'Edit Data Source',
nameLabel: 'Name',
namePlaceholder: 'Enter data source name',
sectionBasic: 'Basic information',
testConnection: 'Test Connection',
testSuccess: 'Connection successful',
testFailed: 'Connection failed',
@@ -5493,11 +5494,13 @@ export default {
feishu: 'Feishu',
notion: 'Notion',
yuque: 'Yuque',
rss: 'RSS / Atom Feed',
},
connectorDesc: {
feishu: 'Sync documents, spreadsheets and files from Feishu Wiki',
notion: 'Sync pages and databases from Notion',
yuque: 'Sync documents from Yuque knowledge bases',
rss: 'Sync articles from RSS / Atom feeds',
},
field: {
appId: 'App ID',
@@ -5506,6 +5509,10 @@ export default {
apiToken: 'API Token',
baseUrl: 'Base URL (optional)',
baseUrlHint: 'Leave empty to use the Yuque public cloud (https://www.yuque.com). For Yuque Enterprise or self-hosted deployments, enter your company domain (e.g. https://your-company.yuque.com).',
feedUrls: 'Feed URLs',
feedUrlsHint: 'One RSS / Atom feed URL per line; multiple feeds are supported.',
authHeaders: 'Custom headers (optional)',
authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx',
},
comingSoon: 'Coming soon',
docHint: 'Get credentials at:',
+7
View File
@@ -5430,6 +5430,7 @@ export default {
editTitle: "데이터 소스 편집",
nameLabel: "이름",
namePlaceholder: "데이터 소스 이름 입력",
sectionBasic: "기본 정보",
testConnection: "연결 테스트",
testSuccess: "연결 성공",
testFailed: "연결 실패",
@@ -5505,11 +5506,13 @@ export default {
feishu: "페이슈 (Feishu)",
notion: "Notion",
yuque: "위큐 (Yuque)",
rss: "RSS / Atom 피드",
},
connectorDesc: {
feishu: "페이슈 위키에서 문서, 스프레드시트, 파일 동기화",
notion: "Notion에서 페이지 및 데이터베이스 동기화",
yuque: "위큐 지식베이스에서 문서 동기화",
rss: "RSS / Atom 피드에서 글 동기화",
},
field: {
appId: "App ID",
@@ -5518,6 +5521,10 @@ export default {
baseUrl: "Base URL",
apiToken: "API Token",
baseUrlHint: "비워두면 Yuque 퍼블릭 클라우드 https://www.yuque.com 를 사용합니다. Yuque Enterprise 또는 사설 배포를 사용하는 경우 기업 도메인(예: https://your-company.yuque.com)을 입력하세요",
feedUrls: "피드 주소",
feedUrlsHint: "한 줄에 하나씩 RSS / Atom 피드 주소를 입력하세요. 여러 개를 함께 입력할 수 있습니다.",
authHeaders: "사용자 지정 헤더 (선택)",
authHeadersHint: "비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx",
},
comingSoon: "곧 지원 예정",
docHint: "다음에서 인증 정보를 받으세요:",
+7
View File
@@ -5252,6 +5252,7 @@ export default {
editTitle: 'Редактировать источник данных',
nameLabel: 'Название',
namePlaceholder: 'Введите название источника данных',
sectionBasic: 'Основная информация',
testConnection: 'Проверить подключение',
testSuccess: 'Подключение успешно',
testFailed: 'Подключение не удалось',
@@ -5327,11 +5328,13 @@ export default {
feishu: 'Feishu (Фэйшу)',
notion: 'Notion',
yuque: 'Yuque (Юйцюэ)',
rss: 'RSS / Atom лента',
},
connectorDesc: {
feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki',
notion: 'Синхронизация страниц и баз данных из Notion',
yuque: 'Синхронизация документов из баз знаний Yuque',
rss: 'Синхронизация статей из лент RSS / Atom',
},
field: {
appId: 'App ID',
@@ -5340,6 +5343,10 @@ export default {
baseUrl: 'Base URL',
apiToken: 'API Token',
baseUrlHint: 'Оставьте пустым, чтобы использовать публичное облако Yuque https://www.yuque.com. Если вы используете Yuque Enterprise или приватное развёртывание, укажите корпоративный домен (например, https://your-company.yuque.com)',
feedUrls: 'Адреса лент',
feedUrlsHint: 'По одному адресу ленты RSS / Atom в строке; можно указать несколько.',
authHeaders: 'Пользовательские заголовки (необязательно)',
authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx',
},
comingSoon: 'Скоро',
docHint: 'Получить учётные данные можно здесь:',
+7
View File
@@ -5436,6 +5436,7 @@ export default {
editTitle: "编辑数据源",
nameLabel: "名称",
namePlaceholder: "输入数据源名称",
sectionBasic: "基本信息",
testConnection: "测试连接",
testSuccess: "连接成功",
testFailed: "连接失败",
@@ -5511,11 +5512,13 @@ export default {
feishu: "飞书",
notion: "Notion",
yuque: "语雀",
rss: "RSS / Atom 订阅",
},
connectorDesc: {
feishu: "同步飞书知识库中的文档、表格、文件",
notion: "同步 Notion 中的页面和数据库",
yuque: "同步语雀知识库中的文档",
rss: "同步 RSS / Atom 订阅源中的文章",
},
field: {
appId: "App ID",
@@ -5524,6 +5527,10 @@ export default {
apiToken: "API Token",
baseUrl: "Base URL(可选)",
baseUrlHint: "留空将使用语雀公有云 https://www.yuque.com;如果你使用的是语雀企业版或私有部署,请填写企业域名(例如 https://your-company.yuque.com)",
feedUrls: "订阅源地址",
feedUrlsHint: "每行一个 RSS / Atom 订阅源地址,支持同时填写多个",
authHeaders: "自定义请求头(可选)",
authHeadersHint: "用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx",
},
comingSoon: "即将支持",
docHint: "在以下地址获取凭证:",
+6 -11
View File
@@ -3948,20 +3948,15 @@ const handleSave = async () => {
.nav-badge {
flex-shrink: 0;
min-width: 18px;
margin-left: 2px;
padding: 0 6px;
border-radius: 10px;
border-radius: 8px;
background: var(--td-bg-color-secondarycontainer);
font-size: 11px;
font-weight: 600;
line-height: 18px;
text-align: center;
color: var(--td-text-color-secondary);
.nav-item.active & {
background: color-mix(in srgb, var(--td-brand-color) 18%, transparent);
color: var(--td-brand-color);
}
font-size: 11px;
line-height: 16px;
font-weight: 500;
text-align: center;
}
.section--prompts {
@@ -1565,20 +1565,15 @@ watch(
.nav-badge {
flex-shrink: 0;
min-width: 18px;
margin-left: 2px;
padding: 0 6px;
border-radius: 10px;
border-radius: 8px;
background: var(--td-bg-color-secondarycontainer);
font-size: 11px;
font-weight: 600;
line-height: 18px;
text-align: center;
color: var(--td-text-color-secondary);
.nav-item.active & {
background: color-mix(in srgb, var(--td-brand-color) 18%, transparent);
color: var(--td-brand-color);
}
font-size: 11px;
line-height: 16px;
font-weight: 500;
text-align: center;
}
.settings-content {
@@ -18,6 +18,7 @@ import {
} from '@/api/datasource'
import SettingDrawer from '@/components/settings/SettingDrawer.vue'
import DataSourceTypeIcon from './DataSourceTypeIcon.vue'
import { getDatasourceIconUrl } from './datasourceIcons'
const props = defineProps<{
kbId: string
@@ -102,10 +103,43 @@ function cancelReplaceCredentials() {
replaceCredentialsMode.value = false
pendingRemoveCredentials.value = false
form.value.config.credentials = {}
rssAuthHeaders.value = []
testResult.value = credentialsConfigured.value ? 'success' : ''
testErrorMsg.value = ''
}
interface CustomHeaderItem {
key: string
value: string
}
const rssAuthHeaders = ref<CustomHeaderItem[]>([])
function serializeAuthHeaders(items: CustomHeaderItem[]): string {
return items
.filter(h => h.key.trim())
.map(h => `${h.key.trim()}: ${h.value}`)
.join('\n')
}
function syncRssAuthHeadersToCredentials() {
if (form.value.type !== 'rss') return
const serialized = serializeAuthHeaders(rssAuthHeaders.value)
if (serialized) {
form.value.config.credentials.auth_headers = serialized
} else {
delete form.value.config.credentials.auth_headers
}
}
function addRssAuthHeader() {
rssAuthHeaders.value.push({ key: '', value: '' })
}
function removeRssAuthHeader(idx: number) {
rssAuthHeaders.value.splice(idx, 1)
}
function needsConnectionTest(): boolean {
return !(isEdit.value && credentialsConfigured.value && !replaceCredentialsMode.value)
}
@@ -289,7 +323,16 @@ interface ConnectorDef {
permissionDocUrl: string
permissionPageUrl: string
requiredPermissions: string[]
fields: { key: string; labelKey: string; placeholder: string; secret?: boolean; optional?: boolean; hintKey?: string }[]
fields: {
key: string
labelKey: string
placeholder: string
secret?: boolean
optional?: boolean
hintKey?: string
multiline?: boolean
fieldType?: 'custom_headers'
}[]
}
const connectorDefs = computed<ConnectorDef[]>(() => [
@@ -336,6 +379,18 @@ const connectorDefs = computed<ConnectorDef[]>(() => [
{ key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://www.yuque.com', optional: true, hintKey: 'datasource.field.baseUrlHint' },
],
},
{
type: 'rss',
available: true,
docUrl: '',
permissionDocUrl: '',
permissionPageUrl: '',
requiredPermissions: [],
fields: [
{ key: 'feed_urls', labelKey: 'datasource.field.feedUrls', placeholder: 'https://example.com/feed.xml', multiline: true, hintKey: 'datasource.field.feedUrlsHint' },
{ key: 'auth_headers', labelKey: 'datasource.field.authHeaders', placeholder: '', optional: true, hintKey: 'datasource.field.authHeadersHint', fieldType: 'custom_headers' },
],
},
])
@@ -366,6 +421,7 @@ watch(visible, async (v) => {
loadedChildrenIds.value = new Set()
loadingChildrenIds.value = new Set()
treeFullyLoaded.value = false
rssAuthHeaders.value = []
if (isEdit.value && props.dataSource) {
// Reset edit/replace toggle every open so an aborted replace doesn't
@@ -416,19 +472,34 @@ watch(
{ deep: true },
)
watch(
rssAuthHeaders,
() => {
syncRssAuthHeadersToCredentials()
if (needsConnectionTest()) {
testResult.value = ''
testErrorMsg.value = ''
}
},
{ deep: true },
)
function selectType(def: ConnectorDef) {
if (!def.available) return
form.value.type = def.type
form.value.name = t(`datasource.connector.${def.type}`)
form.value.config.credentials = {}
rssAuthHeaders.value = []
step.value = 1
}
// --- Test connection (stateless, no DB write) ---
async function testConnection() {
syncRssAuthHeadersToCredentials()
if (!isEdit.value || !credentialsConfigured.value || replaceCredentialsMode.value) {
const fields = currentDef.value?.fields || []
for (const f of fields) {
if (f.optional) continue
if (f.optional || f.fieldType === 'custom_headers') continue
if (!form.value.config.credentials[f.key]) {
MessagePlugin.warning(`${t(f.labelKey)} ${t('datasource.isRequired')}`)
return
@@ -602,13 +673,14 @@ function toggleResource(id: string) {
}
function validateStep1Fields(): boolean {
syncRssAuthHeadersToCredentials()
if (isEdit.value && credentialsConfigured.value && !replaceCredentialsMode.value) {
return true
}
const fields = currentDef.value?.fields || []
for (const f of fields) {
if (f.optional) continue
if (f.optional || f.fieldType === 'custom_headers') continue
if (!form.value.config.credentials[f.key]) {
MessagePlugin.warning(`${t(f.labelKey)} ${t('datasource.isRequired')}`)
return false
@@ -657,6 +729,7 @@ function buildConfigPayload(): Record<string, unknown> {
// the whole submit on failure so we don't leave the row partially saved.
async function commitCredentialsIfNeeded(dsId: string): Promise<boolean> {
if (!isEdit.value || !replaceCredentialsMode.value) return true
syncRssAuthHeadersToCredentials()
const filled = Object.entries(form.value.config.credentials).filter(
([, v]) => typeof v === 'string' ? v !== '' : v != null,
)
@@ -666,6 +739,7 @@ async function commitCredentialsIfNeeded(dsId: string): Promise<boolean> {
credentialsConfigured.value = true
replaceCredentialsMode.value = false
form.value.config.credentials = {}
rssAuthHeaders.value = []
return true
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('credential.saveFailed'))
@@ -719,6 +793,10 @@ async function handleSubmit() {
}
emit('saved')
// Clear before close — otherwise the visible watcher treats the just-saved
// row as an abandoned temp draft and DELETEs it (loadResources creates the
// row early at step 2 with tempDsId).
tempDsId.value = ''
visible.value = false
} catch (e: any) {
MessagePlugin.error(e?.message || e?.error || t('datasource.saveFailed'))
@@ -822,6 +900,7 @@ const drawerConfirmText = computed(() => {
v-model:visible="visible"
:title="drawerTitle"
:description="drawerDescription"
:class="form.type ? `datasource-editor-drawer datasource-editor-drawer--${form.type}` : 'datasource-editor-drawer'"
:hide-footer="step === 0"
:confirm-text="drawerConfirmText"
:confirm-loading="submitting || (step === 1 && testing)"
@@ -830,8 +909,12 @@ const drawerConfirmText = computed(() => {
@confirm="handleDrawerConfirm"
@cancel="handleClose"
>
<template v-if="form.type" #headerIcon>
<DataSourceTypeIcon :type="form.type" :size="20" />
<template v-if="form.type && getDatasourceIconUrl(form.type)" #headerIcon>
<img
:src="getDatasourceIconUrl(form.type)"
:alt="form.type"
class="datasource-header-icon__img"
>
</template>
<template v-if="step === 1" #footer-left>
@@ -879,7 +962,10 @@ const drawerConfirmText = computed(() => {
:key="i"
:class="['ds-step', { active: step === i, done: step > i }]"
>
<span class="ds-step-num">{{ step > i ? '&#10003;' : i + 1 }}</span>
<span class="ds-step-num">
<t-icon v-if="step > i" name="check" class="ds-step-check" />
<template v-else>{{ i + 1 }}</template>
</span>
<span class="ds-step-title">{{ title }}</span>
</div>
</div>
@@ -907,10 +993,10 @@ const drawerConfirmText = computed(() => {
</section>
<!-- Step 1: Credentials -->
<section v-if="step === 1" class="setting-drawer__section">
<template v-if="step === 1">
<div
v-if="currentDef && currentDef.requiredPermissions.length > 0"
class="ds-setup-guide"
class="ds-setup-guide ds-setup-guide--standalone"
>
<button
type="button"
@@ -970,29 +1056,33 @@ const drawerConfirmText = computed(() => {
</div>
</div>
<div class="form-item">
<label class="form-label">{{ t('datasource.nameLabel') }}</label>
<t-input v-model="form.name" :placeholder="t('datasource.namePlaceholder')" />
</div>
<section class="setting-drawer__section">
<h4 class="setting-drawer__section-title">{{ t('datasource.sectionBasic') }}</h4>
<div v-if="currentDef?.docUrl" class="inline-alert">
<t-icon name="info-circle-filled" class="inline-alert__icon" />
<span class="inline-alert__text">{{ t('datasource.docHint') }}</span>
<a
:href="currentDef.docUrl"
target="_blank"
rel="noopener"
class="inline-alert__action doc-link"
>
{{ t('datasource.openDoc') }}
<t-icon name="link" class="link-icon" />
</a>
</div>
<div v-if="currentDef?.docUrl" class="inline-alert">
<t-icon name="info-circle-filled" class="inline-alert__icon" />
<span class="inline-alert__text">{{ t('datasource.docHint') }}</span>
<a
:href="currentDef.docUrl"
target="_blank"
rel="noopener"
class="inline-alert__action doc-link"
>
{{ t('datasource.openDoc') }}
<t-icon name="link" class="link-icon" />
</a>
</div>
<div class="form-item">
<label class="form-label">{{ t('datasource.credentialsLabel') }}</label>
<div class="form-item">
<label class="form-label required">{{ t('datasource.nameLabel') }}</label>
<t-input v-model="form.name" :placeholder="t('datasource.namePlaceholder')" />
</div>
</section>
<template v-if="isEdit && credentialsConfigured && !replaceCredentialsMode">
<section class="setting-drawer__section">
<h4 class="setting-drawer__section-title">{{ t('datasource.credentialsLabel') }}</h4>
<div v-if="isEdit && credentialsConfigured && !replaceCredentialsMode" class="form-item">
<div
class="credential-faux-input"
:class="{ 'is-confirm-remove': pendingRemoveCredentials }"
@@ -1031,54 +1121,103 @@ const drawerConfirmText = computed(() => {
</div>
</template>
</div>
</template>
</div>
<div
v-else-if="isEdit && !credentialsConfigured && !replaceCredentialsMode"
class="credential-faux-input is-empty"
@click="enterReplaceCredentials"
class="form-item"
>
<t-icon name="lock-on" class="credential-status-icon muted" />
<span class="credential-faux-text muted">{{ t('credential.unconfigured') }}</span>
<div class="credential-actions">
<t-button size="small" variant="text" theme="primary" @click.stop="enterReplaceCredentials">
{{ t('credential.configure') }}
</t-button>
<div
class="credential-faux-input is-empty"
@click="enterReplaceCredentials"
>
<t-icon name="lock-on" class="credential-status-icon muted" />
<span class="credential-faux-text muted">{{ t('credential.unconfigured') }}</span>
<div class="credential-actions">
<t-button size="small" variant="text" theme="primary" @click.stop="enterReplaceCredentials">
{{ t('credential.configure') }}
</t-button>
</div>
</div>
</div>
<div v-else-if="credentialsInputVisible" class="credential-edit">
<template v-else-if="credentialsInputVisible">
<div
v-for="field in currentDef?.fields || []"
:key="field.key"
class="credential-field"
class="form-item"
>
<label
v-if="(currentDef?.fields?.length || 0) > 1"
class="credential-field-label"
>
{{ t(field.labelKey) }}
<span v-if="!field.optional" class="required-mark">*</span>
</label>
<t-input
v-model="form.config.credentials[field.key]"
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
:type="field.secret ? 'password' : 'text'"
autocomplete="off"
spellcheck="false"
>
<template v-if="field.secret" #prefix-icon><t-icon name="lock-on" /></template>
</t-input>
<p v-if="field.hintKey" class="form-desc">{{ t(field.hintKey) }}</p>
<template v-if="field.fieldType === 'custom_headers'">
<div class="custom-headers-header">
<label class="form-label" style="margin-bottom: 0;">{{ t(field.labelKey) }}</label>
<t-button variant="text" size="small" theme="primary" @click="addRssAuthHeader">
<template #icon><t-icon name="add" /></template>
{{ t('model.editor.customHeadersAdd') }}
</t-button>
</div>
<p v-if="field.hintKey" class="form-desc custom-headers-desc">{{ t(field.hintKey) }}</p>
<div v-if="rssAuthHeaders.length > 0" class="custom-headers-list">
<div v-for="(item, idx) in rssAuthHeaders" :key="idx" class="custom-header-row">
<t-input
v-model="item.key"
:placeholder="t('model.editor.customHeadersKeyPlaceholder')"
class="custom-header-key"
autocomplete="off"
spellcheck="false"
/>
<t-input
v-model="item.value"
:placeholder="t('model.editor.customHeadersValuePlaceholder')"
class="custom-header-value"
autocomplete="off"
spellcheck="false"
/>
<t-button
variant="text"
shape="square"
size="small"
class="custom-header-remove"
:aria-label="t('common.delete')"
@click="removeRssAuthHeader(idx)"
>
<t-icon name="close" />
</t-button>
</div>
</div>
</template>
<template v-else>
<label class="form-label" :class="{ required: !field.optional }">
{{ t(field.labelKey) }}
</label>
<t-textarea
v-if="field.multiline"
v-model="form.config.credentials[field.key]"
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
:autosize="{ minRows: 2, maxRows: 6 }"
autocomplete="off"
spellcheck="false"
/>
<t-input
v-else
v-model="form.config.credentials[field.key]"
:placeholder="field.placeholder || t('credential.inputPlaceholder')"
:type="field.secret ? 'password' : 'text'"
autocomplete="off"
spellcheck="false"
>
<template v-if="field.secret" #prefix-icon><t-icon name="lock-on" /></template>
</t-input>
<p v-if="field.hintKey" class="form-desc">{{ t(field.hintKey) }}</p>
</template>
</div>
<div v-if="isEdit && replaceCredentialsMode" class="credential-edit-actions">
<t-button size="small" variant="text" @click="cancelReplaceCredentials">
{{ t('common.cancel') }}
</t-button>
</div>
</div>
</div>
</section>
</template>
</section>
</template>
<!-- Step 2: Select resources -->
<section v-if="step === 2" class="setting-drawer__section ds-resource-section">
@@ -1280,7 +1419,7 @@ const drawerConfirmText = computed(() => {
@import './datasource-surface.less';
.ds-steps {
display: flex;
gap: 4px;
gap: 8px;
margin-bottom: 20px;
border-bottom: 1px solid var(--td-component-stroke);
padding-bottom: 14px;
@@ -1289,22 +1428,32 @@ const drawerConfirmText = computed(() => {
.ds-step {
display: flex;
align-items: center;
gap: 6px;
gap: 8px;
flex: 1;
min-width: 0;
font-size: 13px;
color: var(--td-text-color-placeholder);
}
.ds-step-title {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.ds-step.active {
color: var(--td-brand-color);
font-weight: 600;
font-weight: 500;
}
.ds-step.done {
color: var(--td-text-color-secondary);
font-weight: 500;
}
.ds-step-num {
flex-shrink: 0;
width: 22px;
height: 22px;
border-radius: 50%;
@@ -1312,7 +1461,10 @@ const drawerConfirmText = computed(() => {
align-items: center;
justify-content: center;
font-size: 12px;
border: 1px solid currentColor;
font-weight: 600;
border: 1px solid var(--td-component-stroke);
color: var(--td-text-color-placeholder);
background: transparent;
}
.ds-step.active .ds-step-num {
@@ -1322,9 +1474,13 @@ const drawerConfirmText = computed(() => {
}
.ds-step.done .ds-step-num {
background: var(--td-bg-color-secondarycontainer);
color: var(--td-success-color);
border-color: color-mix(in srgb, var(--td-success-color) 35%, var(--td-component-stroke));
background: color-mix(in srgb, var(--td-brand-color) 12%, transparent);
color: var(--td-brand-color);
border-color: transparent;
}
.ds-step-check {
font-size: 14px;
}
.ds-loading-center {
@@ -1422,6 +1578,10 @@ const drawerConfirmText = computed(() => {
gap: 4px;
}
.ds-setup-guide--standalone {
margin-bottom: 4px;
}
.ds-setup-guide__toggle {
display: flex;
align-items: center;
@@ -1594,25 +1754,6 @@ const drawerConfirmText = computed(() => {
margin: 0 2px;
}
.credential-edit {
display: flex;
flex-direction: column;
gap: 10px;
}
.credential-field {
display: flex;
flex-direction: column;
gap: 6px;
}
.credential-field-label {
font-size: 13px;
font-weight: 500;
color: var(--td-text-color-primary);
line-height: 1.4;
}
.credential-edit-actions {
display: flex;
justify-content: flex-end;
@@ -1625,7 +1766,7 @@ const drawerConfirmText = computed(() => {
}
.form-item {
margin-bottom: 16px;
margin-bottom: 0;
}
.form-item--flat {
@@ -1644,11 +1785,15 @@ const drawerConfirmText = computed(() => {
font-weight: 500;
margin-bottom: 6px;
color: var(--td-text-color-primary);
}
line-height: 1.4;
.required-mark {
color: var(--td-error-color);
margin-left: 2px;
&.required::before {
content: '*';
color: var(--td-error-color);
margin-right: 4px;
font-weight: 500;
line-height: 1;
}
}
.form-desc {
@@ -1936,6 +2081,55 @@ const drawerConfirmText = computed(() => {
gap: 16px;
}
.custom-headers-header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 6px;
}
.custom-headers-desc {
margin: 0 0 10px 0;
font-size: 12px;
line-height: 1.5;
color: var(--td-text-color-placeholder);
}
.custom-headers-list {
display: flex;
flex-direction: column;
gap: 8px;
}
.custom-header-row {
display: flex;
align-items: center;
gap: 8px;
.custom-header-key {
flex: 0 0 38%;
}
.custom-header-value {
flex: 1;
}
.custom-header-remove {
flex-shrink: 0;
width: 32px;
height: 32px;
padding: 0;
color: var(--td-text-color-placeholder);
border-radius: 6px;
transition: all 0.18s ease;
&:hover {
background: var(--td-error-color-light);
color: var(--td-error-color);
}
}
}
.ds-empty-retry {
padding: 0;
border: none;
@@ -2001,3 +2195,20 @@ const drawerConfirmText = computed(() => {
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.05);
}
</style>
<!--
Drawer header logo — same white badge as list cards / StorageEngineSettings.
-->
<style lang="less">
.datasource-editor-drawer .setting-drawer__header-icon:has(.datasource-header-icon__img) {
background: var(--td-bg-color-container, #fff);
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
}
.datasource-header-icon__img {
display: block;
width: 24px;
height: 24px;
object-fit: contain;
}
</style>
@@ -202,11 +202,11 @@ onBeforeUnmount(stopPolling)
v-for="ds in dataSources"
:key="ds.id"
:type="canManageDataSource ? 'button' : undefined"
:class="['ds-card', { 'ds-card--clickable': canManageDataSource }]"
:class="['ds-card', `ds-card--${ds.type}`, { 'ds-card--clickable': canManageDataSource }]"
@click="canManageDataSource ? openEdit(ds) : undefined"
>
<div class="ds-card__badge">
<DataSourceTypeIcon :type="ds.type" :size="22" />
<DataSourceTypeIcon :type="ds.type" variant="badge" />
</div>
<div class="ds-card__body">
<div class="ds-card__header">
@@ -452,10 +452,23 @@ onBeforeUnmount(stopPolling)
display: flex;
align-items: center;
justify-content: center;
background: color-mix(in srgb, var(--td-brand-color) 10%, transparent);
margin-top: 1px;
font-size: 15px;
font-weight: 600;
letter-spacing: 0.02em;
background: rgba(7, 192, 95, 0.12);
color: #07c05f;
overflow: hidden;
}
&--feishu .ds-card__badge,
&--notion .ds-card__badge,
&--yuque .ds-card__badge,
&--rss .ds-card__badge {
background: var(--td-bg-color-container, #fff);
box-shadow: inset 0 0 0 1px var(--td-component-stroke);
}
&__body {
flex: 1;
min-width: 0;
@@ -1,20 +1,17 @@
<script setup lang="ts">
import feishuIcon from '@/assets/img/datasource-feishu.ico'
import notionIcon from '@/assets/img/datasource-notion.ico'
import yuqueIcon from '@/assets/img/datasource-yuque.ico'
import { getDatasourceIconUrl, datasourceIconMap } from './datasourceIcons'
const props = withDefaults(defineProps<{
type: string
size?: number
/** inline: 类型选择等小尺寸场景;badge: 嵌入 ds-card__badge 等父级徽章容器 */
variant?: 'inline' | 'badge'
}>(), {
size: 20,
variant: 'inline',
})
const iconMap: Record<string, string> = {
feishu: feishuIcon,
notion: notionIcon,
yuque: yuqueIcon,
}
const iconMap = datasourceIconMap
function fallbackText(type: string) {
switch (type) {
@@ -33,13 +30,15 @@ function fallbackText(type: string) {
<template>
<span
class="ds-type-icon"
:style="{ width: `${size}px`, height: `${size}px` }"
:class="`ds-type-icon--${variant}`"
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
>
<img
v-if="iconMap[type]"
:src="iconMap[type]"
:alt="type"
:style="{ width: `${size}px`, height: `${size}px` }"
class="ds-type-icon__img"
:style="variant === 'inline' ? { width: `${size}px`, height: `${size}px` } : undefined"
>
<span v-else class="ds-type-icon-fallback">{{ fallbackText(type) }}</span>
</span>
@@ -52,18 +51,41 @@ function fallbackText(type: string) {
justify-content: center;
flex-shrink: 0;
overflow: hidden;
}
.ds-type-icon--inline {
border-radius: 6px;
background: var(--td-bg-color-component);
}
.ds-type-icon img {
.ds-type-icon--inline .ds-type-icon__img {
display: block;
object-fit: cover;
object-fit: contain;
}
.ds-type-icon-fallback {
.ds-type-icon--inline .ds-type-icon-fallback {
font-size: 11px;
font-weight: 600;
color: var(--td-text-color-placeholder);
}
.ds-type-icon--badge {
width: 100%;
height: 100%;
background: transparent;
}
.ds-type-icon--badge .ds-type-icon__img {
display: block;
width: 24px;
height: 24px;
object-fit: contain;
}
.ds-type-icon--badge .ds-type-icon-fallback {
font-size: 15px;
font-weight: 600;
letter-spacing: 0.02em;
color: inherit;
}
</style>
@@ -0,0 +1,15 @@
import feishuIcon from '@/assets/img/datasource-feishu.ico'
import notionIcon from '@/assets/img/datasource-notion.ico'
import yuqueIcon from '@/assets/img/datasource-yuque.ico'
import rssIcon from '@/assets/img/datasource-rss.svg'
export const datasourceIconMap: Record<string, string> = {
feishu: feishuIcon,
notion: notionIcon,
yuque: yuqueIcon,
rss: rssIcon,
}
export function getDatasourceIconUrl(type: string): string | undefined {
return datasourceIconMap[type]
}
+6
View File
@@ -3,6 +3,7 @@ module github.com/Tencent/WeKnora
go 1.26.0
require (
codeberg.org/readeck/go-readability/v2 v2.1.2
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.1
github.com/PuerkitoBio/goquery v1.12.0
@@ -35,6 +36,7 @@ require (
github.com/mark3labs/mcp-go v0.52.0
github.com/milvus-io/milvus/client/v2 v2.6.4
github.com/minio/minio-go/v7 v7.1.0
github.com/mmcdole/gofeed v1.3.0
github.com/neo4j/neo4j-go-driver/v6 v6.0.0
github.com/ollama/ollama v0.23.2
github.com/open-dingtalk/dingtalk-stream-sdk-go v0.9.1
@@ -176,6 +178,7 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
@@ -183,6 +186,7 @@ require (
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/flatbuffers v25.12.19+incompatible // indirect
@@ -194,6 +198,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/itlightning/dateparse v0.2.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -227,6 +232,7 @@ require (
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
+12
View File
@@ -1324,6 +1324,8 @@ cloud.google.com/go/workflows v1.12.1/go.mod h1:5A95OhD/edtOhQd/O741NSfIMezNTbCw
cloud.google.com/go/workflows v1.12.2/go.mod h1:+OmBIgNqYJPVggnMo9nqmizW0qEXHhmnAzK/CnBqsHc=
cloud.google.com/go/workflows v1.12.3/go.mod h1:fmOUeeqEwPzIU81foMjTRQIdwQHADi/vEr1cx9R1m5g=
cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w=
codeberg.org/readeck/go-readability/v2 v2.1.2 h1:JBrdyYJBRPMBbodLM1b5KxCSDH+JqCkGcuVRD7ICBAw=
codeberg.org/readeck/go-readability/v2 v2.1.2/go.mod h1:Ut31sW4osSrJPR3T8eQslMh4+jbwimXqn0w0ReCT+PU=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
@@ -1773,6 +1775,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -1809,6 +1813,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
@@ -2026,6 +2032,8 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
github.com/itlightning/dateparse v0.2.1 h1:AB0NJTyI0HYcerEUMovKZOiQVBg1mBPxgAnWQwzLP6g=
github.com/itlightning/dateparse v0.2.1/go.mod h1:xHlmL8lT0L9JIBlaKotRwsoDYpKJskXpiU9ZwbbSkNA=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -2212,6 +2220,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+4
View File
@@ -57,6 +57,7 @@ import (
"github.com/Tencent/WeKnora/internal/datasource"
feishuConnector "github.com/Tencent/WeKnora/internal/datasource/connector/feishu"
notionConnector "github.com/Tencent/WeKnora/internal/datasource/connector/notion"
rssConnector "github.com/Tencent/WeKnora/internal/datasource/connector/rss"
yuqueConnector "github.com/Tencent/WeKnora/internal/datasource/connector/yuque"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/handler"
@@ -1365,6 +1366,9 @@ func initConnectorRegistry() (*datasource.ConnectorRegistry, error) {
if err := registry.Register(yuqueConnector.NewConnector()); err != nil {
errs = errors.Join(errs, fmt.Errorf("register yuque connector: %w", err))
}
if err := registry.Register(rssConnector.NewConnector()); err != nil {
errs = errors.Join(errs, fmt.Errorf("register rss connector: %w", err))
}
// Future connectors will be registered here:
// if err := registry.Register(confluenceConnector.NewConnector()); err != nil { ... }
+121
View File
@@ -0,0 +1,121 @@
package rss
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"time"
readability "codeberg.org/readeck/go-readability/v2"
"github.com/Tencent/WeKnora/internal/utils"
)
const (
// requestTimeout bounds a single feed or article fetch.
requestTimeout = 20 * time.Second
// maxFeedSize caps the feed body to avoid memory blowups on hostile feeds.
maxFeedSize = 10 * 1024 * 1024 // 10 MB
// maxArticleSize caps a single article page body.
maxArticleSize = 5 * 1024 * 1024 // 5 MB
// defaultUserAgent is sent on every request; some feeds reject empty UAs.
defaultUserAgent = "Mozilla/5.0 (compatible; WeKnora-RSS/1.0; +https://weknora.weixin.qq.com)"
)
// client performs SSRF-safe HTTP fetches with optional custom headers.
type client struct {
httpClient *http.Client
headers map[string]string
}
func newClient(headers map[string]string) *client {
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
cfg.Timeout = requestTimeout
return &client{
httpClient: utils.NewSSRFSafeHTTPClient(cfg),
headers: headers,
}
}
// fetch retrieves rawURL with SSRF validation and size limiting. The user's
// custom headers are applied first, then a default User-Agent / Accept if the
// user did not override them.
func (c *client) fetch(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
return nil, fmt.Errorf("URL rejected: %w", err)
}
if _, err := url.Parse(rawURL); err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, err
}
for k, v := range c.headers {
req.Header.Set(k, v)
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", defaultUserAgent)
}
if req.Header.Get("Accept") == "" {
req.Header.Set("Accept",
"application/rss+xml, application/atom+xml, application/xml, text/xml, application/json, text/html;q=0.9, */*;q=0.8")
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize))
if err != nil {
return nil, fmt.Errorf("read body failed: %w", err)
}
return body, nil
}
// fetchFeed retrieves the raw bytes of a feed document.
func (c *client) fetchFeed(ctx context.Context, feedURL string) ([]byte, error) {
return c.fetch(ctx, feedURL, maxFeedSize)
}
// extractArticle fetches an article page and returns the readability-cleaned
// main content as HTML, plus the extracted title (may be empty). Returns an
// error if the page can't be fetched or no readable content is found, so the
// caller can fall back to feed-provided content.
func (c *client) extractArticle(ctx context.Context, articleURL string) (contentHTML, title string, err error) {
body, err := c.fetch(ctx, articleURL, maxArticleSize)
if err != nil {
return "", "", err
}
pageURL, _ := url.Parse(articleURL)
article, err := readability.FromReader(bytes.NewReader(body), pageURL)
if err != nil {
return "", "", fmt.Errorf("readability parse: %w", err)
}
if article.Node == nil {
return "", "", fmt.Errorf("no readable content extracted")
}
var buf bytes.Buffer
if err := article.RenderHTML(&buf); err != nil {
return "", "", fmt.Errorf("render article html: %w", err)
}
return buf.String(), article.Title(), nil
}
@@ -0,0 +1,308 @@
package rss
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"time"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/mmcdole/gofeed"
)
// Compile-time proof that *Connector satisfies the datasource.Connector interface.
var _ datasource.Connector = (*Connector)(nil)
// Connector implements datasource.Connector for RSS/Atom/JSON feeds.
type Connector struct{}
// NewConnector creates a new RSS connector.
func NewConnector() *Connector { return &Connector{} }
// Type returns the connector type identifier.
func (c *Connector) Type() string { return types.ConnectorTypeRSS }
// Validate verifies that every configured feed URL is reachable and parses as
// a valid feed.
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
cfg, err := parseConfig(config)
if err != nil {
return err
}
cli := newClient(cfg.parseHeaders())
parser := gofeed.NewParser()
for _, feedURL := range cfg.feedURLList() {
data, err := cli.fetchFeed(ctx, feedURL)
if err != nil {
return fmt.Errorf("fetch feed %s: %w", feedURL, err)
}
if _, err := parser.Parse(bytes.NewReader(data)); err != nil {
return fmt.Errorf("parse feed %s: %w", feedURL, err)
}
}
return nil
}
// ResolveResourceAncestors has nothing to do: feeds are a flat list with no
// nesting, so a selection has no ancestors to reveal.
func (c *Connector) ResolveResourceAncestors(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]string, error) {
return []string{}, nil
}
// ListResources returns one resource per configured feed URL. The feed is
// fetched so the resource can carry its real title; a feed that fails to fetch
// still appears (named by URL) with an error note, so the user can deselect it
// instead of the whole listing failing.
func (c *Connector) ListResources(
ctx context.Context, config *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
// Feeds are flat: a lazy-load request for a specific parent has nothing extra.
if parentID != "" {
return []types.Resource{}, nil
}
cfg, err := parseConfig(config)
if err != nil {
return nil, err
}
cli := newClient(cfg.parseHeaders())
parser := gofeed.NewParser()
feedURLs := cfg.feedURLList()
out := make([]types.Resource, 0, len(feedURLs))
for _, feedURL := range feedURLs {
res := types.Resource{
ExternalID: feedURL,
Type: "feed",
Name: feedURL,
URL: feedURL,
}
data, err := cli.fetchFeed(ctx, feedURL)
if err != nil {
logger.Warnf(ctx, "[RSS] list: fetch %s failed: %v", feedURL, err)
res.Description = "fetch failed: " + err.Error()
out = append(out, res)
continue
}
feed, err := parser.Parse(bytes.NewReader(data))
if err != nil {
logger.Warnf(ctx, "[RSS] list: parse %s failed: %v", feedURL, err)
res.Description = "parse failed: " + err.Error()
out = append(out, res)
continue
}
if title := strings.TrimSpace(feed.Title); title != "" {
res.Name = title
}
res.Description = strings.TrimSpace(feed.Description)
if feed.Link != "" {
res.URL = feed.Link
}
if feed.UpdatedParsed != nil {
res.ModifiedAt = *feed.UpdatedParsed
}
res.Metadata = map[string]interface{}{"item_count": len(feed.Items)}
out = append(out, res)
}
return out, nil
}
// FetchAll performs a full sync of the specified feeds (or all configured feeds
// when resourceIDs is empty).
func (c *Connector) FetchAll(
ctx context.Context, config *types.DataSourceConfig, resourceIDs []string,
) ([]types.FetchedItem, error) {
items, _, err := c.walk(ctx, config, resourceIDs, nil, false)
return items, err
}
// FetchIncremental returns only items whose fingerprint changed since the prior
// cursor. Deletions are intentionally not emitted (feeds drop old items as a
// matter of course).
func (c *Connector) FetchIncremental(
ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor,
) ([]types.FetchedItem, *types.SyncCursor, error) {
var prev *rssCursor
if cursor != nil && cursor.ConnectorCursor != nil {
var p rssCursor
b, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(b, &p)
prev = &p
}
items, newCursor, err := c.walk(ctx, config, config.ResourceIDs, prev, true)
if err != nil {
return nil, nil, err
}
cursorMap := make(map[string]interface{})
b, _ := json.Marshal(newCursor)
_ = json.Unmarshal(b, &cursorMap)
return items, &types.SyncCursor{
LastSyncTime: newCursor.LastSyncTime,
ConnectorCursor: cursorMap,
}, nil
}
// walk is the shared implementation for FetchAll / FetchIncremental.
//
// When incremental is true, items whose fingerprint matches the prior cursor
// are skipped (no article re-fetch). The returned cursor always reflects the
// full current item set so a later sync can detect changes.
func (c *Connector) walk(
ctx context.Context,
config *types.DataSourceConfig,
resourceIDs []string,
prev *rssCursor,
incremental bool,
) ([]types.FetchedItem, *rssCursor, error) {
cfg, err := parseConfig(config)
if err != nil {
return nil, nil, err
}
// Default to all configured feeds when no explicit selection was made.
feedURLs := resourceIDs
if len(feedURLs) == 0 {
feedURLs = cfg.feedURLList()
}
cli := newClient(cfg.parseHeaders())
parser := gofeed.NewParser()
newCursor := &rssCursor{
LastSyncTime: time.Now().UTC(),
FeedItems: make(map[string]map[string]string),
}
var out []types.FetchedItem
for _, feedURL := range feedURLs {
data, err := cli.fetchFeed(ctx, feedURL)
if err != nil {
return nil, nil, fmt.Errorf("fetch feed %s: %w", feedURL, err)
}
feed, err := parser.Parse(bytes.NewReader(data))
if err != nil {
return nil, nil, fmt.Errorf("parse feed %s: %w", feedURL, err)
}
newCursor.FeedItems[feedURL] = make(map[string]string)
var prevItems map[string]string
if incremental && prev != nil {
prevItems = prev.FeedItems[feedURL]
}
var kept, skipped int
for _, item := range feed.Items {
if item == nil {
continue
}
itemID := firstNonEmpty(item.GUID, item.Link, item.Title)
if itemID == "" {
continue
}
feedContent := firstNonEmpty(item.Content, item.Description)
fp := itemFingerprint(item.UpdatedParsed, item.PublishedParsed, feedContent)
newCursor.FeedItems[feedURL][itemID] = fp
if incremental && prevItems != nil && prevItems[itemID] == fp {
skipped++
continue
}
kept++
out = append(out, c.buildItem(ctx, cli, feed, item, feedURL, itemID, feedContent))
}
logger.Infof(ctx, "[RSS] feed %s: items=%d fetched=%d skipped=%d",
feedURL, len(feed.Items), kept, skipped)
}
if !incremental {
return out, newCursor, nil
}
return out, newCursor, nil
}
// buildItem assembles a FetchedItem for a single feed entry, resolving the
// best available content (full-text article > feed content) and converting it
// to Markdown.
func (c *Connector) buildItem(
ctx context.Context,
cli *client,
feed *gofeed.Feed,
item *gofeed.Item,
feedURL, itemID, feedContent string,
) types.FetchedItem {
title := firstNonEmpty(item.Title, "untitled")
// Prefer full article text; fall back to feed-provided content on failure.
contentHTML := feedContent
if strings.TrimSpace(item.Link) != "" {
if articleHTML, articleTitle, err := cli.extractArticle(ctx, item.Link); err == nil {
contentHTML = articleHTML
if item.Title == "" && articleTitle != "" {
title = articleTitle
}
} else {
logger.Warnf(ctx, "[RSS] full-text fetch failed for %s (using feed content): %v", item.Link, err)
}
}
content := htmlToMarkdown(contentHTML)
updatedAt := time.Now().UTC()
switch {
case item.UpdatedParsed != nil && !item.UpdatedParsed.IsZero():
updatedAt = *item.UpdatedParsed
case item.PublishedParsed != nil && !item.PublishedParsed.IsZero():
updatedAt = *item.PublishedParsed
}
author := ""
if item.Author != nil {
author = item.Author.Name
}
return types.FetchedItem{
ExternalID: itemID,
Title: title,
Content: []byte(content),
ContentType: "text/markdown",
FileName: sanitizeFileName(title) + ".md",
URL: item.Link,
UpdatedAt: updatedAt,
SourceResourceID: feedURL,
Metadata: map[string]string{
"channel": types.ChannelRSS,
"feed_url": feedURL,
"feed_title": feed.Title,
"guid": item.GUID,
"link": item.Link,
"author": author,
},
}
}
// htmlToMarkdown converts HTML to Markdown, returning the trimmed original on
// conversion failure so we never silently drop content.
func htmlToMarkdown(html string) string {
if strings.TrimSpace(html) == "" {
return ""
}
md, err := htmltomd.ConvertString(html)
if err != nil || strings.TrimSpace(md) == "" {
return strings.TrimSpace(html)
}
return strings.TrimSpace(md)
}
@@ -0,0 +1,250 @@
package rss
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
)
// TestMain whitelists loopback for SSRF so the httptest servers (127.0.0.1)
// are reachable. Production keeps the default strict SSRF policy.
func TestMain(m *testing.M) {
_ = os.Setenv("SSRF_WHITELIST", "127.0.0.1,::1")
utils.ResetSSRFWhitelistForTest()
code := m.Run()
os.Exit(code)
}
const longArticleBody = `<p>This is the first paragraph of a reasonably long article that the ` +
`readability extractor should detect as the main content of the page. It contains ` +
`enough words to clear the minimum content threshold used by the algorithm.</p>` +
`<p>The second paragraph continues the discussion with more sentences so that the ` +
`scoring heuristics confidently select this block over navigation and footer noise.</p>` +
`<p>A third paragraph adds further substance, ensuring the article is unmistakably ` +
`the dominant readable region of the document under test.</p>`
// fakeFeed spins up an httptest server serving an RSS feed and article pages.
type fakeFeed struct {
server *httptest.Server
feedTitle string
itemContent string // optional <description>/content for items
}
func newFakeFeed(t *testing.T) *fakeFeed {
t.Helper()
f := &fakeFeed{feedTitle: "Test Feed"}
mux := http.NewServeMux()
mux.HandleFunc("/article/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprintf(w, `<!DOCTYPE html><html><head><title>%s</title></head>`+
`<body><nav>menu</nav><article><h1>Heading</h1>%s</article><footer>foot</footer></body></html>`,
"Full "+r.URL.Path, longArticleBody)
})
mux.HandleFunc("/feed.xml", func(w http.ResponseWriter, r *http.Request) {
if auth := r.Header.Get("X-Test-Auth"); f.itemContent == "needs-auth" && auth != "secret" {
w.WriteHeader(http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/rss+xml")
base := "http://" + r.Host
desc := "summary fallback"
fmt.Fprintf(w, `<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>%s</title>
<link>%s</link>
<description>A test feed</description>
<item>
<title>Article One</title>
<link>%s/article/a1</link>
<guid>guid-1</guid>
<pubDate>Mon, 02 Jan 2006 15:04:05 GMT</pubDate>
<description>%s</description>
</item>
<item>
<title>Article Two</title>
<link>%s/article/a2</link>
<guid>guid-2</guid>
<pubDate>Tue, 03 Jan 2006 15:04:05 GMT</pubDate>
<description>%s</description>
</item>
</channel></rss>`, f.feedTitle, base, base, desc, base, desc)
})
f.server = httptest.NewServer(mux)
t.Cleanup(f.server.Close)
return f
}
func (f *fakeFeed) feedURL() string { return f.server.URL + "/feed.xml" }
func makeConfig(feedURLs string, headers string) *types.DataSourceConfig {
creds := map[string]interface{}{"feed_urls": feedURLs}
if headers != "" {
creds["auth_headers"] = headers
}
return &types.DataSourceConfig{
Type: types.ConnectorTypeRSS,
Credentials: creds,
}
}
func TestConnector_Type(t *testing.T) {
if NewConnector().Type() != types.ConnectorTypeRSS {
t.Fatalf("Type() = %q, want %q", NewConnector().Type(), types.ConnectorTypeRSS)
}
}
func TestParseConfig_RequiresFeedURLs(t *testing.T) {
if _, err := parseConfig(makeConfig(" ", "")); err == nil {
t.Fatal("expected error when feed_urls is blank")
}
}
func TestConfig_FeedURLList_SplitsAndDedupes(t *testing.T) {
cfg := &Config{FeedURLs: "https://a.com/f, https://b.com/f\nhttps://a.com/f\n"}
got := cfg.feedURLList()
want := []string{"https://a.com/f", "https://b.com/f"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Fatalf("feedURLList = %v, want %v", got, want)
}
}
func TestConfig_ParseHeaders(t *testing.T) {
cfg := &Config{AuthHeaders: "Authorization: Bearer x\nX-Foo: bar \nbroken-line\n: noname"}
got := cfg.parseHeaders()
if got["Authorization"] != "Bearer x" {
t.Errorf("Authorization = %q", got["Authorization"])
}
if got["X-Foo"] != "bar" {
t.Errorf("X-Foo = %q", got["X-Foo"])
}
if len(got) != 2 {
t.Errorf("expected 2 headers, got %d: %v", len(got), got)
}
}
func TestConnector_Validate_Success(t *testing.T) {
f := newFakeFeed(t)
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err != nil {
t.Fatalf("Validate error: %v", err)
}
}
func TestConnector_Validate_PrivateFeedWithHeader(t *testing.T) {
f := newFakeFeed(t)
f.itemContent = "needs-auth"
// Without the header → 401.
if err := NewConnector().Validate(context.Background(), makeConfig(f.feedURL(), "")); err == nil {
t.Fatal("expected error without auth header")
}
// With the header → success.
if err := NewConnector().Validate(
context.Background(), makeConfig(f.feedURL(), "X-Test-Auth: secret"),
); err != nil {
t.Fatalf("Validate with header error: %v", err)
}
}
func TestConnector_ListResources(t *testing.T) {
f := newFakeFeed(t)
res, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "")
if err != nil {
t.Fatalf("ListResources error: %v", err)
}
if len(res) != 1 {
t.Fatalf("expected 1 resource, got %d", len(res))
}
if res[0].ExternalID != f.feedURL() {
t.Errorf("ExternalID = %q, want %q", res[0].ExternalID, f.feedURL())
}
if res[0].Name != "Test Feed" {
t.Errorf("Name = %q, want %q", res[0].Name, "Test Feed")
}
// Non-empty parentID → empty (feeds are flat).
children, err := NewConnector().ListResources(context.Background(), makeConfig(f.feedURL(), ""), "feed-x")
if err != nil {
t.Fatalf("ListResources(parent) error: %v", err)
}
if len(children) != 0 {
t.Fatalf("expected no children, got %d", len(children))
}
}
func TestConnector_FetchAll_FullTextMarkdown(t *testing.T) {
f := newFakeFeed(t)
items, err := NewConnector().FetchAll(context.Background(), makeConfig(f.feedURL(), ""), nil)
if err != nil {
t.Fatalf("FetchAll error: %v", err)
}
if len(items) != 2 {
t.Fatalf("expected 2 items, got %d", len(items))
}
it := items[0]
if it.ExternalID != "guid-1" {
t.Errorf("ExternalID = %q, want guid-1", it.ExternalID)
}
if it.ContentType != "text/markdown" {
t.Errorf("ContentType = %q, want text/markdown", it.ContentType)
}
if it.Metadata["channel"] != types.ChannelRSS {
t.Errorf("channel = %q, want %q", it.Metadata["channel"], types.ChannelRSS)
}
// Full text from the article page should be present (not the short summary).
if !strings.Contains(string(it.Content), "first paragraph") {
t.Errorf("expected full article text in content, got: %q", string(it.Content))
}
if !strings.HasSuffix(it.FileName, ".md") {
t.Errorf("FileName = %q, want .md suffix", it.FileName)
}
}
func TestConnector_FetchIncremental_SkipsUnchanged(t *testing.T) {
f := newFakeFeed(t)
cfg := makeConfig(f.feedURL(), "")
cfg.ResourceIDs = []string{f.feedURL()}
// First sync: everything is new.
items, cursor, err := NewConnector().FetchIncremental(context.Background(), cfg, nil)
if err != nil {
t.Fatalf("first FetchIncremental error: %v", err)
}
if len(items) != 2 {
t.Fatalf("expected 2 items on first sync, got %d", len(items))
}
if cursor == nil {
t.Fatal("expected non-nil cursor")
}
// Second sync with the returned cursor: nothing changed → no items.
items2, _, err := NewConnector().FetchIncremental(context.Background(), cfg, cursor)
if err != nil {
t.Fatalf("second FetchIncremental error: %v", err)
}
if len(items2) != 0 {
t.Fatalf("expected 0 items on unchanged second sync, got %d", len(items2))
}
}
func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
got, err := NewConnector().ResolveResourceAncestors(
context.Background(), makeConfig("https://a.com/f", ""), []string{"x"},
)
if err != nil {
t.Fatalf("ResolveResourceAncestors error: %v", err)
}
if len(got) != 0 {
t.Fatalf("expected empty ancestors, got %v", got)
}
}
+188
View File
@@ -0,0 +1,188 @@
// Package rss implements the RSS/Atom data source connector for WeKnora.
//
// It syncs articles from one or more RSS/Atom/JSON feeds into a WeKnora
// knowledge base. Each configured feed URL is treated as a selectable
// resource; each feed item becomes a knowledge entry whose body is the
// article's full text rendered as Markdown.
//
// Capabilities:
// - Private feeds: optional custom HTTP headers (e.g. "Authorization: Bearer …")
// are attached to every request, both to the feed and to article pages.
// - Full text: when an item exposes a link, the article page is fetched and
// run through a readability extractor; the cleaned HTML is converted to
// Markdown. Feed-provided content (content:encoded / description) is used
// as a fallback.
// - Incremental: a per-feed fingerprint (item updated/published time, or a
// content hash when no timestamp is present) detects changed items so only
// new/updated articles are re-fetched. Deletions are NOT synced — feeds
// routinely drop old items, which would otherwise look like deletions.
//
// All outbound requests go through the SSRF-safe HTTP client so a malicious
// feed cannot redirect WeKnora to internal services.
package rss
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
)
// Config holds RSS-specific configuration parsed from DataSourceConfig.Credentials.
//
// Both fields live under Credentials (rather than Settings) because the
// existing data-source editor UI only renders credential inputs, and because
// AuthHeaders may carry secrets that must be encrypted at rest. FeedURLs are
// not secret but ride along for the same single-input flow.
type Config struct {
// FeedURLs is a newline- or comma-separated list of feed URLs.
FeedURLs string `json:"feed_urls"`
// AuthHeaders is an optional newline-separated list of custom request
// headers in "Name: Value" form, applied to every outbound request.
AuthHeaders string `json:"auth_headers,omitempty"`
}
// parseConfig extracts and validates RSS configuration from the credentials map.
func parseConfig(config *types.DataSourceConfig) (*Config, error) {
if config == nil {
return nil, fmt.Errorf("%w: config is nil", datasource.ErrInvalidConfig)
}
credBytes, err := json.Marshal(config.Credentials)
if err != nil {
return nil, fmt.Errorf("marshal credentials: %w", err)
}
var cfg Config
if err := json.Unmarshal(credBytes, &cfg); err != nil {
return nil, fmt.Errorf("parse rss credentials: %w", err)
}
if len(cfg.feedURLList()) == 0 {
return nil, fmt.Errorf("%w: feed_urls is required", datasource.ErrInvalidCredentials)
}
return &cfg, nil
}
// feedURLList splits FeedURLs on newlines and commas, trims, dedupes (order
// preserved), and drops blanks.
func (c *Config) feedURLList() []string {
if c == nil {
return nil
}
raw := strings.FieldsFunc(c.FeedURLs, func(r rune) bool {
return r == '\n' || r == '\r' || r == ','
})
seen := make(map[string]struct{}, len(raw))
out := make([]string, 0, len(raw))
for _, u := range raw {
u = strings.TrimSpace(u)
if u == "" {
continue
}
if _, ok := seen[u]; ok {
continue
}
seen[u] = struct{}{}
out = append(out, u)
}
return out
}
// parseHeaders turns the newline-separated "Name: Value" AuthHeaders blob into
// a map. Lines without a colon, or with an empty name, are skipped.
func (c *Config) parseHeaders() map[string]string {
if c == nil || strings.TrimSpace(c.AuthHeaders) == "" {
return nil
}
headers := make(map[string]string)
for _, line := range strings.Split(c.AuthHeaders, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
idx := strings.Index(line, ":")
if idx <= 0 {
continue
}
name := strings.TrimSpace(line[:idx])
value := strings.TrimSpace(line[idx+1:])
if name == "" {
continue
}
headers[name] = value
}
if len(headers) == 0 {
return nil
}
return headers
}
// rssCursor stores incremental sync state.
//
// FeedItems maps feedURL → itemID → fingerprint, where fingerprint is the
// item's updated/published timestamp (RFC3339) or, when no timestamp exists,
// a "h:<sha256-prefix>" content hash. An item is considered unchanged when its
// fingerprint matches the stored one.
type rssCursor struct {
LastSyncTime time.Time `json:"last_sync_time"`
FeedItems map[string]map[string]string `json:"feed_items,omitempty"`
}
// itemFingerprint returns a change-detection token for a feed item. It prefers
// the explicit updated/published timestamp; absent that, it hashes the content
// so edits to timestamp-less items are still detected.
func itemFingerprint(updated, published *time.Time, content string) string {
if updated != nil && !updated.IsZero() {
return updated.UTC().Format(time.RFC3339)
}
if published != nil && !published.IsZero() {
return published.UTC().Format(time.RFC3339)
}
sum := sha256.Sum256([]byte(content))
return "h:" + hex.EncodeToString(sum[:])[:16]
}
// firstNonEmpty returns the first non-empty trimmed string among the args.
func firstNonEmpty(values ...string) string {
for _, v := range values {
if strings.TrimSpace(v) != "" {
return v
}
}
return ""
}
// sanitizeFileName removes characters invalid in filenames and truncates to a
// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
func sanitizeFileName(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return "untitled"
}
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
"\n", " ", "\r", " ", "\t", " ",
)
result := strings.TrimSpace(replacer.Replace(name))
if result == "" {
return "untitled"
}
const maxBytes = 200
if len(result) > maxBytes {
result = result[:maxBytes]
for len(result) > 0 {
r, size := utf8.DecodeLastRuneInString(result)
if r != utf8.RuneError || size != 1 {
break
}
result = result[:len(result)-1]
}
}
return result
}
+8
View File
@@ -53,6 +53,10 @@ func (r *fakeDataSourceRepo) Update(_ context.Context, ds *types.DataSource) err
return nil
}
func (r *fakeDataSourceRepo) UpdateSyncState(ctx context.Context, ds *types.DataSource) error {
return r.Update(ctx, ds)
}
func (r *fakeDataSourceRepo) Delete(_ context.Context, id string) error {
r.mu.Lock()
defer r.mu.Unlock()
@@ -117,6 +121,10 @@ func (r *fakeSyncLogRepo) Update(_ context.Context, log *types.SyncLog) error {
return nil
}
func (r *fakeSyncLogRepo) UpdateResult(ctx context.Context, log *types.SyncLog) error {
return r.Update(ctx, log)
}
func (r *fakeSyncLogRepo) CancelPendingByDataSource(_ context.Context, dsID string) error {
return nil
}
+1
View File
@@ -30,6 +30,7 @@ const (
ChannelIM = "im" // Generic IM channel
ChannelNotion = "notion" // Notion
ChannelYuque = "yuque" // Yuque (语雀)
ChannelRSS = "rss" // RSS / Atom feed
)
// Knowledge parse status constants