fix(feishu): resilient large-wiki sync — retry, streaming resume, failure visibility

Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136:
5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run).
Three compounding causes, fixed in layers:

1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export
   APIs are aggressively rate limited; a thousand-doc sync issues tens of
   thousands of calls, and a single 429/5xx burst failed whole swathes of docs.
   Add exponential backoff honouring Retry-After to doRequest (JSON) and
   downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off.

2. The batch path fetched the entire tree into memory before ingesting, so a
   sync that hit the 2h task timeout lost all progress and restarted from
   scratch, and a transient per-document export failure advanced the cursor and
   skipped that doc permanently. Add an optional StreamingConnector interface,
   implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint,
   bounding memory to one item and persisting a per-node edit-time cursor at
   page boundaries. The service dispatches to it via type assertion; connectors
   without it fall back to FetchAll / FetchIncremental unchanged (non-breaking).
   A resumed/incremental run skips nodes already recorded at their current edit
   time (converges), a failed fetch does not advance the cursor (retried next
   run instead of dropped), and partial listing failures emit error items and
   carry prior edit times forward instead of aborting the sync.

3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…)
   were skipped with no item, error or log ("13 docs synced only 3"). FetchAll
   now tallies discovered/fetched/failed/skipped per resource with a per-type
   skip breakdown and a progress line every 100 nodes. The backend already
   persisted per-item errors in SyncLog.result.errors but the frontend type
   omitted the field; declare `result` on SyncLog and render up to 50 failed
   documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU.
   result.errors is a jsonb column shipped in every list response, so it is
   capped at 100 samples server-side (recordSyncError); the exact failure total
   comes from the bounded items_failed counter, which the drawer uses for its
   count and "N more" line so a sync that fails thousands of docs stays bounded.
   Each sample is a structured, localisable SyncItemError instead of a raw
   "title: ...api error: status=500 body={json}" dump that repeated the title
   and leaked status/body/log_id. The connector (feishuFailure) classifies the
   error into a stable i18n code (+ the feishu code as a param); the service
   builds {title, code, params, message}; the drawer localises via vue-i18n to
   the viewer's language (zh/en/ko/ru), falling back to message. A custom
   UnmarshalJSON keeps old string-form logs readable. The banner is localised
   from items_failed too. Raw errors stay in the server logs. Follows the
   two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised
   summaries in the UI; raw detail in logs. Transient errors say "will retry"
   (the cursor is retained); auth/permission errors point at the fix instead.

Tests: client_retry_test (backoff), tally_test (accounting), connector_stream /
datasource_stream (streaming dispatch + checkpoint), connector_convergence
(timeout + transient-failure resume converges to full coverage; proven to catch
#2136 via bug injection), datasource_result_cap (error-sample cap + structured
localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code
classification), sync_item_error (legacy-string + structured back-compat) — all
in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu
harness gated by the feishu_integration build tag AND env vars, never run in CI.
This commit is contained in:
nullkey
2026-07-21 00:20:57 +08:00
committed by lyingbug
parent 74cdd7cdd0
commit c695125dce
23 changed files with 2565 additions and 125 deletions
+24
View File
@@ -25,6 +25,29 @@ export interface DataSource {
latest_sync_log?: SyncLog
}
/**
* One user-facing failure sample. The backend sends a stable i18n `code`
* (+ interpolation `params`) so the UI localises it to the viewer's language;
* `message` is a fallback when no i18n key exists (old logs decode into it).
*/
export interface SyncItemError {
title?: string
code?: string
params?: Record<string, string>
message?: string
}
export interface SyncResultDetail {
total?: number
created?: number
updated?: number
deleted?: number
skipped?: number
failed?: number
/** Per-item failure samples (capped); localised in the sync-log drawer. */
errors?: SyncItemError[]
}
export interface SyncLog {
id: string
data_source_id: string
@@ -38,6 +61,7 @@ export interface SyncLog {
items_skipped: number
items_failed: number
error_message: string
result?: SyncResultDetail
}
export interface ConnectorMeta {
+13
View File
@@ -6216,6 +6216,19 @@ export default {
startTime: 'Started',
endTime: 'Finished',
duration: 'Duration',
failedItems: 'Failed documents',
failedItemsMore: '{n} more failed documents not shown',
docsFailedSummary: '{n} document(s) failed to sync',
},
syncError: {
feishu_auth_or_permission: 'Authentication or permission error; check credentials and app permissions',
feishu_rate_limited: 'Feishu API rate limited; will retry on the next sync',
feishu_timeout: 'Export or request timed out; will retry on the next sync',
feishu_server_unavailable: 'Feishu service temporarily unavailable; will retry on the next sync',
feishu_api_error: 'Feishu API error (code={code}); will retry on the next sync',
feishu_api_error_generic: 'Feishu API error; will retry on the next sync',
sync_failed: 'Sync failed; will retry on the next sync',
ingest_failed: 'Ingest failed; see server logs',
},
connector: {
feishu: 'Feishu',
+13
View File
@@ -6221,6 +6221,19 @@ export default {
startTime: "시작 시간",
endTime: "종료 시간",
duration: "소요 시간",
failedItems: "실패한 문서",
failedItemsMore: "표시되지 않은 실패 문서 {n}개",
docsFailedSummary: "문서 {n}개 동기화 실패",
},
syncError: {
feishu_auth_or_permission: "인증 또는 권한 부족: 자격 증명과 앱 권한을 확인하세요",
feishu_rate_limited: "Feishu API 요청 제한, 다음 동기화 시 다시 시도합니다",
feishu_timeout: "문서 내보내기 또는 요청 시간 초과, 다음 동기화 시 다시 시도합니다",
feishu_server_unavailable: "Feishu 서비스가 일시적으로 사용 불가, 다음 동기화 시 다시 시도합니다",
feishu_api_error: "Feishu API 오류(code={code}), 다음 동기화 시 다시 시도합니다",
feishu_api_error_generic: "Feishu API 오류, 다음 동기화 시 다시 시도합니다",
sync_failed: "동기화 실패, 다음 동기화 시 다시 시도합니다",
ingest_failed: "가져오기 실패, 서버 로그를 확인하세요",
},
connector: {
feishu: "페이슈 (Feishu)",
+13
View File
@@ -6042,6 +6042,19 @@ export default {
startTime: 'Время начала',
endTime: 'Время окончания',
duration: 'Длительность',
failedItems: 'Документы с ошибкой',
failedItemsMore: 'Ещё {n} документов с ошибкой не показаны',
docsFailedSummary: 'Не удалось синхронизировать документов: {n}',
},
syncError: {
feishu_auth_or_permission: 'Ошибка аутентификации или недостаточно прав; проверьте учётные данные и права приложения',
feishu_rate_limited: 'Превышен лимит запросов к API Feishu; повтор при следующей синхронизации',
feishu_timeout: 'Тайм-аут экспорта или запроса; повтор при следующей синхронизации',
feishu_server_unavailable: 'Сервис Feishu временно недоступен; повтор при следующей синхронизации',
feishu_api_error: 'Ошибка API Feishu (code={code}); повтор при следующей синхронизации',
feishu_api_error_generic: 'Ошибка API Feishu; повтор при следующей синхронизации',
sync_failed: 'Ошибка синхронизации; повтор при следующей синхронизации',
ingest_failed: 'Ошибка импорта; смотрите логи сервера',
},
connector: {
feishu: 'Feishu (Фэйшу)',
+13
View File
@@ -6241,6 +6241,19 @@ export default {
startTime: "开始时间",
endTime: "结束时间",
duration: "耗时",
failedItems: "失败文档",
failedItemsMore: "还有 {n} 个失败文档未显示",
docsFailedSummary: "{n} 个文档同步失败",
},
syncError: {
feishu_auth_or_permission: "应用鉴权或权限不足,请检查凭证与应用权限",
feishu_rate_limited: "飞书接口限流,下次同步时将重试",
feishu_timeout: "文档导出或请求超时,下次同步时将重试",
feishu_server_unavailable: "飞书服务暂时不可用,下次同步时将重试",
feishu_api_error: "飞书接口错误(code={code}),下次同步时将重试",
feishu_api_error_generic: "飞书接口错误,下次同步时将重试",
sync_failed: "同步失败,下次同步时将重试",
ingest_failed: "入库失败,详情见服务日志",
},
connector: {
feishu: "飞书",
@@ -1,7 +1,7 @@
<script setup lang="ts">
import { ref, watch, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { getSyncLogs, type SyncLog } from '@/api/datasource'
import { getSyncLogs, type SyncLog, type SyncItemError } from '@/api/datasource'
const props = defineProps<{
dataSourceId: string
@@ -131,6 +131,30 @@ function hasPills(log: SyncLog) {
return log.items_created > 0 || log.items_updated > 0 || log.items_deleted > 0 || log.items_skipped > 0 || log.items_failed > 0
}
// Cap the per-item failure list so a sync that failed thousands of documents
// doesn't render an unbounded wall of text; the remainder is summarised.
const FAILED_ITEMS_CAP = 50
function failedItems(log: SyncLog): SyncItemError[] {
return (log.result?.errors || []).slice(0, FAILED_ITEMS_CAP)
}
// Render one failure sample. The backend sends a stable i18n `code` (+ params)
// so the reason is localised to the viewer's language; `message` is the fallback
// for old logs / codes this client doesn't know. The document title is kept
// separate and prefixed as "title — reason".
function formatSyncError(e: SyncItemError): string {
let reason = ''
if (e.code) {
const key = `datasource.syncError.${e.code}`
const localised = t(key, (e.params || {}) as Record<string, unknown>)
reason = localised === key ? (e.message || e.code) : localised
} else {
reason = e.message || ''
}
return e.title ? (reason ? `${e.title} — ${reason}` : e.title) : reason
}
// Group logs by date
const groupedLogs = computed(() => {
const groups: { date: string; logs: SyncLog[] }[] = []
@@ -253,9 +277,29 @@ const groupedLogs = computed(() => {
<span class="detail-label">{{ t('datasource.logMetric.total') }}</span>
<span>{{ log.items_total }}</span>
</div>
<div v-if="log.error_message" class="tl-error">
<!-- Localised failure summary; raw error_message only for a
pure infra failure with no per-document detail. -->
<div v-if="log.items_failed > 0" class="tl-error">
{{ t('datasource.logDetail.docsFailedSummary', { n: log.items_failed }) }}
</div>
<div v-else-if="log.error_message" class="tl-error">
{{ log.error_message }}
</div>
<!-- Per-item failures: which documents failed and why.
The true count is items_failed (a bounded int); result.errors
is only a capped sample the backend retains for display. -->
<div v-if="failedItems(log).length" class="tl-failed">
<div class="tl-failed-title">
{{ t('datasource.logDetail.failedItems') }} ({{ log.items_failed }})
</div>
<div v-for="(e, i) in failedItems(log)" :key="i" class="tl-failed-item" :title="formatSyncError(e)">
{{ formatSyncError(e) }}
</div>
<div v-if="log.items_failed > failedItems(log).length" class="tl-failed-more">
{{ t('datasource.logDetail.failedItemsMore', { n: (log.items_failed - failedItems(log).length) }) }}
</div>
</div>
</div>
</div>
</div>
@@ -513,6 +557,39 @@ const groupedLogs = computed(() => {
word-break: break-word;
}
/* --- Per-item failure list --- */
.tl-failed {
margin-top: 8px;
display: flex;
flex-direction: column;
gap: 2px;
}
.tl-failed-title {
font-size: 11px;
font-weight: 600;
color: var(--td-error-color);
margin-bottom: 2px;
}
.tl-failed-item {
font-size: 11px;
line-height: 1.5;
color: var(--td-text-color-secondary);
padding: 2px 8px;
border-left: 2px solid var(--td-error-color-3);
background: var(--td-bg-color-container);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.tl-failed-more {
font-size: 11px;
color: var(--td-text-color-placeholder);
padding: 2px 8px;
}
/* --- Drawer header --- */
.logs-drawer-header {
display: flex;
@@ -0,0 +1,67 @@
package service
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
)
// A sync that fails thousands of documents must not accumulate an unbounded
// result.Errors slice: that slice is persisted as jsonb and shipped in every
// sync-log list response, so an uncapped list means multi-MB rows and payloads.
// The accurate count lives in result.Failed (a bounded int), so Errors only
// needs to retain a sample for display (Tencent/WeKnora#2136 / #1262).
func TestApplyFetchedItem_CapsErrorSampleButKeepsAccurateFailedCount(t *testing.T) {
s := &DataSourceService{}
ds := &types.DataSource{}
result := &types.SyncResult{}
ctx := context.Background()
const failures = 5000
for i := 0; i < failures; i++ {
s.applyFetchedItem(ctx, ds, &types.FetchedItem{
ExternalID: "node",
Title: "doc",
Metadata: map[string]string{"error": "export failed: rate limited"},
}, nil, result)
}
assert.Equal(t, failures, result.Failed,
"Failed must count every failure regardless of the error-sample cap")
assert.LessOrEqual(t, len(result.Errors), maxSyncResultErrors,
"Errors sample must be capped so the persisted jsonb stays bounded")
assert.Equal(t, maxSyncResultErrors, len(result.Errors),
"with more failures than the cap, the sample should be exactly the cap")
}
// The user-facing sample must be a structured SyncItemError carrying the
// connector's stable i18n code + params (so the frontend localises it), the
// title as a separate field, and no raw API body/log_id — that stays in logs.
func TestApplyFetchedItem_ProducesLocalisableStructuredError(t *testing.T) {
s := &DataSourceService{}
ds := &types.DataSource{}
result := &types.SyncResult{}
s.applyFetchedItem(context.Background(), ds, &types.FetchedItem{
ExternalID: "nt1",
Title: "季度报告",
Metadata: map[string]string{
// raw, for logs only
"error": `export 季度报告 (docx): feishu api error: status=500 body={"code":1663,"error":{"log_id":"20260"}}`,
// classification, for a localisable UI sample
"error_reason_code": "feishu_api_error",
"error_reason_code_value": "1663",
"error_reason": "Feishu API error (code=1663); will retry automatically",
},
}, nil, result)
assert.Equal(t, 1, len(result.Errors))
got := result.Errors[0]
assert.Equal(t, "季度报告", got.Title, "title is a separate field, not embedded in the message")
assert.Equal(t, "feishu_api_error", got.Code, "stable i18n code drives frontend localisation")
assert.Equal(t, "1663", got.Params["code"], "feishu error code is passed as an interpolation param")
assert.NotContains(t, got.Message, "body=", "raw API body must not reach the UI sample")
assert.NotContains(t, got.Message, "log_id", "raw log_id must not reach the UI sample")
}
@@ -625,6 +625,13 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
return err
}
// Streaming path: connectors that support it interleave fetch→ingest→
// checkpoint so a large sync bounds memory and resumes after a timeout
// instead of restarting (Tencent/WeKnora#2136). Others fall back below.
if sc, ok := connector.(datasource.StreamingConnector); ok {
return s.processSyncStreaming(ctx, sc, ds, syncLog, config, payload, wasPaused)
}
// Fetch items based on sync mode
var items []types.FetchedItem
var nextCursor *types.SyncCursor
@@ -697,56 +704,11 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
// Auto-tag: find or create a tag for this data source so synced items are easily identifiable
autoTagIDs := []string{}
autoTagName := ds.Name
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, autoTagName); tagErr != nil {
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", autoTagName, tagErr)
} else if autoTag != nil {
autoTagIDs = append(autoTagIDs, autoTag.ID)
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", autoTagName, autoTag.ID)
}
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
for _, item := range items {
if item.IsDeleted {
if ds.SyncDeletions {
// Count only — actual KB deletion is intentionally not performed.
// Users manage knowledge removal explicitly via the KB UI to avoid
// accidental data loss from connector misdetection or reconfiguration.
result.Deleted++
}
continue
}
if len(item.Content) == 0 && item.URL == "" {
// Check if this is an error item from the connector (failed to fetch content)
if errMsg, hasErr := item.Metadata["error"]; hasErr {
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
result.Failed++
result.Errors = append(result.Errors, fmt.Sprintf("%s: %s", item.Title, errMsg))
} else {
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
result.Skipped++
}
continue
}
isUpdate, err := s.ingestItem(ctx, ds, &item, autoTagIDs)
if err != nil {
// Duplicate file/URL is not a failure — count as skipped
var dupErr *types.DuplicateKnowledgeError
if errors.As(err, &dupErr) {
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
result.Skipped++
} else {
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
result.Failed++
result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", item.Title, err))
}
} else if isUpdate {
result.Updated++
} else {
result.Created++
}
item := item
s.applyFetchedItem(ctx, ds, &item, autoTagIDs, result)
}
resultJSON, _ := result.ToJSON()
@@ -769,7 +731,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
syncStatus = types.SyncLogStatusPartial
syncErrorMessage = fmt.Sprintf("Some feeds failed: %s", strings.Join(fetchWarnings, "; "))
for _, w := range fetchWarnings {
result.Errors = append(result.Errors, w)
result.Errors = append(result.Errors, types.SyncItemError{Message: w})
}
resultJSON, _ = result.ToJSON()
}
@@ -781,6 +743,253 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
return nil
}
// resolveAutoTagIDs finds or creates the per-data-source tag applied to every
// synced item so results are identifiable in the KB. A tag failure is
// non-fatal: the sync proceeds untagged.
func (s *DataSourceService) resolveAutoTagIDs(ctx context.Context, ds *types.DataSource) []string {
autoTagIDs := []string{}
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, ds.Name); tagErr != nil {
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", ds.Name, tagErr)
} else if autoTag != nil {
autoTagIDs = append(autoTagIDs, autoTag.ID)
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", ds.Name, autoTag.ID)
}
return autoTagIDs
}
// maxSyncResultErrors bounds the per-item error sample retained in
// SyncResult.Errors. That slice is persisted as jsonb and returned in every
// sync-log list response, so an unbounded list on a sync that fails thousands of
// documents means multi-MB DB rows and payloads. The accurate failure count
// lives in SyncResult.Failed (a bounded int); this list only keeps a sample for
// display (Tencent/WeKnora#2136 / #1262).
const maxSyncResultErrors = 100
// recordSyncError appends an error sample to result.Errors, capped at
// maxSyncResultErrors. Callers still increment result.Failed for the exact count.
func recordSyncError(result *types.SyncResult, item types.SyncItemError) {
if len(result.Errors) < maxSyncResultErrors {
result.Errors = append(result.Errors, item)
}
}
// fetchFailureSyncError maps a connector error item into a structured, user-
// facing sample. Connectors that classify their errors (Feishu) provide a stable
// i18n code + params via metadata so the frontend localises it to the viewer's
// language; the raw status/body/log_id never leaves the server logs. Connectors
// without codes keep the raw text as a Message fallback. Best practice per
// Airbyte/Fivetran/Onyx: humanised, actionable, localised UI; raw detail in logs.
func fetchFailureSyncError(item *types.FetchedItem, rawMsg string) types.SyncItemError {
e := types.SyncItemError{Title: item.Title}
if code := item.Metadata["error_reason_code"]; code != "" {
e.Code = code
if v := item.Metadata["error_reason_code_value"]; v != "" {
e.Params = map[string]string{"code": v}
}
e.Message = item.Metadata["error_reason"] // fallback if the client lacks the key
} else {
e.Message = rawMsg
}
return e
}
// applyFetchedItem writes a single fetched item into the knowledge base and
// updates result counters. It is the shared core of the batch loop and the
// streaming handler so item classification (deleted / empty / ingest outcome)
// stays identical across both fetch paths.
func (s *DataSourceService) applyFetchedItem(
ctx context.Context, ds *types.DataSource, item *types.FetchedItem,
tagIDs []string, result *types.SyncResult,
) {
if item.IsDeleted {
if ds.SyncDeletions {
// Count only — actual KB deletion is intentionally not performed.
// Users manage knowledge removal explicitly via the KB UI to avoid
// accidental data loss from connector misdetection or reconfiguration.
result.Deleted++
}
return
}
if len(item.Content) == 0 && item.URL == "" {
// Check if this is an error item from the connector (failed to fetch content)
if errMsg, hasErr := item.Metadata["error"]; hasErr {
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
result.Failed++
recordSyncError(result, fetchFailureSyncError(item, errMsg))
} else {
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
result.Skipped++
}
return
}
isUpdate, err := s.ingestItem(ctx, ds, item, tagIDs)
if err != nil {
// Duplicate file/URL is not a failure — count as skipped
var dupErr *types.DuplicateKnowledgeError
if errors.As(err, &dupErr) {
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
result.Skipped++
} else {
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
result.Failed++
recordSyncError(result, types.SyncItemError{
Title: item.Title,
Code: "ingest_failed",
Message: "Ingest failed; see server logs",
})
}
} else if isUpdate {
result.Updated++
} else {
result.Created++
}
}
// streamStartCursor decides which cursor a streaming fetch should resume from.
// A user-triggered full sync on its first attempt drops the cursor so every
// item is re-fetched; a retried full sync (attempt > 0) and every incremental
// sync resume from the last persisted checkpoint so a timed-out run converges
// instead of restarting from scratch.
func streamStartCursor(ds *types.DataSource, forceFull bool, attempt int) (*types.SyncCursor, error) {
if forceFull && attempt == 0 {
return nil, nil
}
return ds.ParseSyncCursor()
}
// streamSyncHandler adapts a streaming fetch to the knowledge-base ingest path.
// Emit ingests each item as it arrives (bounding memory) and Checkpoint persists
// the connector cursor plus live progress counts at page boundaries.
type streamSyncHandler struct {
svc *DataSourceService
ds *types.DataSource
tagIDs []string
result *types.SyncResult
syncLog *types.SyncLog
}
// Emit ingests one streamed item. A canceled context aborts the stream so the
// connector stops fetching; per-item ingest failures are recorded in result and
// do NOT abort (matching the batch loop, which never fails the whole sync for
// one bad document).
func (h *streamSyncHandler) Emit(ctx context.Context, item types.FetchedItem) error {
if err := ctx.Err(); err != nil {
return err
}
h.result.Total++
h.svc.applyFetchedItem(ctx, h.ds, &item, h.tagIDs, h.result)
return nil
}
// Checkpoint persists the connector cursor onto the data source and mirrors the
// running counts into the sync log so progress survives a crash and the UI can
// reflect a long sync mid-flight instead of jumping from 0 to done.
func (h *streamSyncHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
if cursor == nil {
return nil
}
cursorJSON, err := cursor.ToJSON()
if err != nil {
return err
}
h.ds.LastSyncCursor = cursorJSON
if err := h.svc.dsRepo.UpdateSyncState(ctx, h.ds); err != nil {
return err
}
// Best-effort live progress; a failure here must not abort the sync.
h.syncLog.ItemsTotal = h.result.Total
h.syncLog.ItemsCreated = h.result.Created
h.syncLog.ItemsUpdated = h.result.Updated
h.syncLog.ItemsDeleted = h.result.Deleted
h.syncLog.ItemsSkipped = h.result.Skipped
h.syncLog.ItemsFailed = h.result.Failed
if err := h.svc.syncLogRepo.UpdateResult(ctx, h.syncLog); err != nil {
logger.Warnf(ctx, "failed to persist sync log progress at checkpoint: %v", err)
}
return nil
}
// processSyncStreaming runs a sync through a StreamingConnector, ingesting each
// item as it arrives and checkpointing progress so the run is memory-bounded and
// resumable after a timeout.
func (s *DataSourceService) processSyncStreaming(
ctx context.Context, sc datasource.StreamingConnector,
ds *types.DataSource, syncLog *types.SyncLog,
config *types.DataSourceConfig, payload types.DataSourceSyncPayload, wasPaused bool,
) error {
// Tenant + auto-tag setup must precede fetching because the stream ingests
// each item on the fly.
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
if err != nil {
logger.Errorf(ctx, "failed to get tenant info: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
types.SyncLogStatusFailed, fmt.Sprintf("Failed to get tenant info: %v", err), wasPaused)
return err
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
forceFull := payload.ForceFull || ds.SyncMode == types.SyncModeFull
attempt, _ := asynq.GetRetryCount(ctx)
startCursor, err := streamStartCursor(ds, forceFull, attempt)
if err != nil {
logger.Errorf(ctx, "failed to parse sync cursor: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
types.SyncLogStatusFailed, fmt.Sprintf("Invalid cursor: %v", err), wasPaused)
return err
}
result := &types.SyncResult{}
handler := &streamSyncHandler{svc: s, ds: ds, tagIDs: autoTagIDs, result: result, syncLog: syncLog}
nextCursor, fetchErr := sc.FetchStream(ctx, config, startCursor, handler)
if fetchErr != nil {
// Progress so far is already checkpointed onto ds.LastSyncCursor; leave
// it in place so the Asynq retry resumes from there. Persist counts.
logger.Errorf(ctx, "streaming fetch failed: %v", fetchErr)
resultJSON, _ := result.ToJSON()
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON,
types.SyncLogStatusFailed, fmt.Sprintf("Fetch failed: %v", fetchErr), wasPaused)
return fetchErr
}
resultJSON, _ := result.ToJSON()
if err := allFetchedItemsFailedError(result); err != nil {
logger.Errorf(ctx, "streaming sync failed while processing fetched items: %v", err)
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
return err
}
// Persist the final cursor for the next incremental sync.
if nextCursor != nil {
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
ds.LastSyncCursor = cursorJSON
}
}
ds.LastSyncAt = timePtr(time.Now().UTC())
// Surface per-document failures as a partial sync (not silent success), so
// the sync-log drawer's failure detail explains which docs didn't make it —
// the visibility gap behind "status normal but not everything syncs"
// (Tencent/WeKnora#2136). Failed nodes were not advanced in the cursor, so
// the next run retries them.
status := types.SyncLogStatusSuccess
errMsg := ""
if result.Failed > 0 {
status = types.SyncLogStatusPartial
errMsg = fmt.Sprintf("%d document(s) failed to sync", result.Failed)
}
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, status, errMsg, wasPaused)
logger.Infof(ctx, "streaming sync completed: ds=%s created=%d updated=%d deleted=%d skipped=%d failed=%d",
payload.DataSourceID, result.Created, result.Updated, result.Deleted, result.Skipped, result.Failed)
return nil
}
func (s *DataSourceService) updateSyncRunResult(
ctx context.Context,
ds *types.DataSource,
@@ -832,7 +1041,7 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
detail := ""
if len(result.Errors) > 0 {
detail = result.Errors[0]
detail = result.Errors[0].Display()
const maxDetailLen = 500
if len(detail) > maxDetailLen {
detail = detail[:maxDetailLen] + "..."
@@ -153,7 +153,7 @@ func TestAllFetchedItemsFailedError(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 2,
Failed: 2,
Errors: []string{"doc one: export failed"},
Errors: []types.SyncItemError{{Message: "doc one: export failed"}},
})
require.Error(t, err)
assert.Contains(t, err.Error(), "all fetched items failed during sync (2/2)")
@@ -181,7 +181,7 @@ func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {
err := allFetchedItemsFailedError(&types.SyncResult{
Total: 1,
Failed: 1,
Errors: []string{strings.Repeat("x", 600)},
Errors: []types.SyncItemError{{Message: strings.Repeat("x", 600)}},
})
require.Error(t, err)
assert.LessOrEqual(t, len(err.Error()), 560)
@@ -0,0 +1,117 @@
package service
import (
"context"
"encoding/json"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recordingDSRepo captures UpdateSyncState calls so checkpoint persistence can
// be asserted without a database.
type recordingDSRepo struct {
kbDeleteDSRepo
updated []*types.DataSource
}
func (r *recordingDSRepo) UpdateSyncState(_ context.Context, ds *types.DataSource) error {
// Snapshot the fields a checkpoint is expected to persist.
cp := *ds
r.updated = append(r.updated, &cp)
return nil
}
func makeConnectorCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) types.JSON {
t.Helper()
inner := map[string]interface{}{"space_node_times": spaceNodeTimes}
b, err := json.Marshal(&types.SyncCursor{ConnectorCursor: inner})
require.NoError(t, err)
return types.JSON(b)
}
// A fresh full sync (ForceFull, first attempt) must ignore any recorded cursor
// and re-fetch everything. A retry of that same task (attempt > 0) must instead
// resume from the checkpointed cursor so it converges instead of restarting.
func TestStreamStartCursor_ForceFullFirstAttemptDropsCursor(t *testing.T) {
ds := &types.DataSource{
LastSyncCursor: makeConnectorCursor(t, map[string]map[string]string{"space1": {"nt1": "100"}}),
}
fresh, err := streamStartCursor(ds, true /*forceFull*/, 0 /*attempt*/)
require.NoError(t, err)
assert.Nil(t, fresh, "fresh ForceFull must drop the cursor to re-fetch everything")
retry, err := streamStartCursor(ds, true /*forceFull*/, 1 /*attempt*/)
require.NoError(t, err)
require.NotNil(t, retry, "a retried ForceFull must resume from the checkpoint")
assert.NotNil(t, retry.ConnectorCursor["space_node_times"])
}
// Incremental sync always resumes from the recorded cursor regardless of attempt.
func TestStreamStartCursor_IncrementalKeepsCursor(t *testing.T) {
ds := &types.DataSource{
LastSyncCursor: makeConnectorCursor(t, map[string]map[string]string{"space1": {"nt1": "100"}}),
}
cur, err := streamStartCursor(ds, false /*forceFull*/, 0)
require.NoError(t, err)
require.NotNil(t, cur)
assert.NotNil(t, cur.ConnectorCursor["space_node_times"])
}
func newStreamHandler(svc *DataSourceService, ds *types.DataSource, result *types.SyncResult, syncLog *types.SyncLog) *streamSyncHandler {
return &streamSyncHandler{svc: svc, ds: ds, result: result, syncLog: syncLog}
}
// Emit routes items through the same classification as the batch loop: deleted
// items are counted, and connector-reported failures (an item carrying only a
// Metadata["error"]) land in result.Failed with a message — never silently lost.
func TestStreamHandler_EmitClassifiesDeletedAndFailed(t *testing.T) {
ds := &types.DataSource{ID: "ds-1", Type: types.ConnectorTypeFeishu, SyncDeletions: true}
result := &types.SyncResult{}
h := newStreamHandler(&DataSourceService{}, ds, result, &types.SyncLog{})
require.NoError(t, h.Emit(context.Background(), types.FetchedItem{ExternalID: "gone", IsDeleted: true}))
require.NoError(t, h.Emit(context.Background(), types.FetchedItem{
ExternalID: "bad", Title: "Broken Doc",
Metadata: map[string]string{"error": "export failed"},
}))
assert.Equal(t, 1, result.Deleted)
assert.Equal(t, 1, result.Failed)
require.Len(t, result.Errors, 1)
assert.Equal(t, "Broken Doc", result.Errors[0].Title)
assert.Contains(t, result.Errors[0].Message, "export failed")
}
// A canceled context aborts the stream: Emit returns the context error so the
// connector stops fetching instead of burning API budget on a doomed run.
func TestStreamHandler_EmitAbortsOnCanceledContext(t *testing.T) {
h := newStreamHandler(&DataSourceService{}, &types.DataSource{}, &types.SyncResult{}, &types.SyncLog{})
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := h.Emit(ctx, types.FetchedItem{ExternalID: "x", Content: []byte("data"), FileName: "x.md"})
require.Error(t, err)
assert.ErrorIs(t, err, context.Canceled)
}
// Checkpoint persists the connector cursor onto the data source so a crash
// after it keeps the progress made so far.
func TestStreamHandler_CheckpointPersistsCursor(t *testing.T) {
dsRepo := &recordingDSRepo{}
svc := &DataSourceService{dsRepo: dsRepo, syncLogRepo: &processSyncSyncLogRepo{logs: map[string]*types.SyncLog{}}}
ds := &types.DataSource{ID: "ds-1"}
result := &types.SyncResult{Created: 3}
syncLog := &types.SyncLog{ID: "log-1"}
h := newStreamHandler(svc, ds, result, syncLog)
cursor := &types.SyncCursor{ConnectorCursor: map[string]interface{}{
"space_node_times": map[string]map[string]string{"space1": {"nt1": "100"}},
}}
require.NoError(t, h.Checkpoint(context.Background(), cursor))
require.Len(t, dsRepo.updated, 1)
assert.NotEmpty(t, dsRepo.updated[0].LastSyncCursor, "checkpoint must persist the cursor JSON")
}
+42
View File
@@ -51,6 +51,48 @@ type Connector interface {
FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error)
}
// StreamHandler receives items and progress checkpoints emitted during a
// streaming fetch. The service implements it to ingest each item as it arrives
// (bounding memory to one item instead of the whole wiki) and to persist the
// connector cursor at page boundaries, so a sync that times out mid-traversal
// resumes from the last checkpoint instead of restarting from scratch
// (Tencent/WeKnora#2136).
type StreamHandler interface {
// Emit ingests a single fetched item. Returning an error aborts the
// stream: the connector stops fetching and propagates the error, since a
// failed ingest means the sync is failing and further API calls are wasted.
Emit(ctx context.Context, item types.FetchedItem) error
// Checkpoint persists the cursor reached so far. The cursor is only valid
// for the duration of the call (the connector may keep mutating its backing
// maps afterwards), so implementations must serialize it synchronously.
//
// The cursor MUST be a complete resumable snapshot, not a delta: resuming
// from it must reproduce all progress so far. This is what lets the service
// treat a checkpoint as a safe restart point and, for a full sync, drop the
// prior baseline without losing already-synced state.
Checkpoint(ctx context.Context, cursor *types.SyncCursor) error
}
// StreamingConnector is an optional interface. Connectors that implement it let
// the service interleave fetch→ingest→checkpoint so a large sync persists
// incrementally and resumes after a timeout, rather than holding every item in
// memory and losing all progress on retry. Connectors that do not implement it
// fall back to FetchAll / FetchIncremental unchanged.
type StreamingConnector interface {
Connector
// FetchStream walks the configured resources starting from cursor (nil =
// from the beginning / full sync), calling h.Emit for each changed item and
// h.Checkpoint at page boundaries. It returns the final cursor for the next
// sync. Nodes already recorded in cursor at their current edit time are
// skipped, which is what makes a resumed sync converge.
FetchStream(
ctx context.Context, config *types.DataSourceConfig,
cursor *types.SyncCursor, h StreamHandler,
) (*types.SyncCursor, error)
}
// ConnectorRegistry manages the registration and lookup of available connectors
type ConnectorRegistry struct {
connectors map[string]Connector
+213 -57
View File
@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
@@ -117,58 +118,161 @@ func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
return c.tokenCache, nil
}
// doRequest executes an authenticated API request and decodes the JSON response.
// Retry policy shared by doRequest (JSON API calls) and downloadRawBytes (file
// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off.
const (
feishuMaxRetries = 3
feishuMax5xxRetries = 1
feishuRetry5xxDelay = 2 * time.Second
)
var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
// doRequest executes an authenticated API request and decodes the JSON response,
// retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive
// export/wiki APIs are aggressively rate limited, and a thousand-document sync
// issues tens of thousands of calls; without backoff a single 429 burst used to
// fail whole swathes of documents silently. 429 responses honour Retry-After;
// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx).
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
const (
maxRetries = feishuMaxRetries
max5xxRetries = feishuMax5xxRetries
retry5xxDelay = feishuRetry5xxDelay
)
backoff := feishuRetryBackoff
token, err := c.getTenantAccessToken(ctx)
if err != nil {
return err
}
var bodyReader io.Reader
var bodyBytes []byte
if body != nil {
bodyBytes, err := json.Marshal(body)
bodyBytes, err = json.Marshal(body)
if err != nil {
return fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(bodyBytes)
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", "Bearer "+token)
var lastErr error
logger.Infof(ctx, "[Feishu] %s %s", method, path)
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("execute request: %w", err)
}
defer resp.Body.Close()
// Read body once for logging + decoding
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read response body: %w", err)
}
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
}
if result != nil {
if err := json.Unmarshal(respBody, result); err != nil {
return fmt.Errorf("decode response: %w", err)
for attempt := 0; attempt <= maxRetries; attempt++ {
var bodyReader io.Reader
if bodyBytes != nil {
bodyReader = bytes.NewReader(bodyBytes)
}
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", "Bearer "+token)
if attempt == 0 {
logger.Infof(ctx, "[Feishu] %s %s", method, path)
} else {
logger.Infof(ctx, "[Feishu] %s %s (retry %d/%d)", method, path, attempt, maxRetries)
}
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = fmt.Errorf("execute request: %w", err)
if attempt < maxRetries {
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
return sErr
}
continue
}
return lastErr
}
respBody, readErr := io.ReadAll(resp.Body)
resp.Body.Close()
if readErr != nil {
lastErr = fmt.Errorf("read response body: %w", readErr)
if attempt < maxRetries {
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
return sErr
}
continue
}
return lastErr
}
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
if resp.StatusCode == http.StatusTooManyRequests {
wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)])
lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500))
if attempt < maxRetries {
if sErr := sleepCtx(ctx, wait); sErr != nil {
return sErr
}
continue
}
return lastErr
}
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
lastErr = fmt.Errorf("feishu server error: status=%d body=%s", resp.StatusCode, truncate(string(respBody), 500))
if attempt < max5xxRetries {
if sErr := sleepCtx(ctx, retry5xxDelay); sErr != nil {
return sErr
}
continue
}
return lastErr
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
}
if result != nil {
if err := json.Unmarshal(respBody, result); err != nil {
return fmt.Errorf("decode response: %w", err)
}
}
return nil
}
return nil
return lastErr
}
// parseRetryAfter interprets a Retry-After header value (seconds) into a wait
// duration, coercing 0/negative to a short delay and falling back when absent
// or unparseable.
func parseRetryAfter(header string, fallback time.Duration) time.Duration {
if header == "" {
return fallback
}
secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64)
if err != nil {
return fallback
}
if secs <= 0 {
return 100 * time.Millisecond
}
return time.Duration(secs * float64(time.Second))
}
// sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the
// context ends first so retries abort promptly on task cancellation/timeout.
func sleepCtx(ctx context.Context, d time.Duration) error {
if d <= 0 {
return ctx.Err()
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
// truncate truncates a string to maxLen and appends "..." if truncated.
@@ -554,31 +658,83 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err
}
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("create download request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+token)
var lastErr error
logger.Infof(ctx, "[Feishu] download GET %s", path)
for attempt := 0; attempt <= feishuMaxRetries; attempt++ {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("create download request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+token)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("download request: %w", err)
}
defer resp.Body.Close()
if attempt == 0 {
logger.Infof(ctx, "[Feishu] download GET %s", path)
} else {
logger.Infof(ctx, "[Feishu] download GET %s (retry %d/%d)", path, attempt, feishuMaxRetries)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = fmt.Errorf("download request: %w", err)
if attempt < feishuMaxRetries {
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
return nil, sErr
}
continue
}
return nil, lastErr
}
if resp.StatusCode == http.StatusTooManyRequests {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)])
lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500))
if attempt < feishuMaxRetries {
if sErr := sleepCtx(ctx, wait); sErr != nil {
return nil, sErr
}
continue
}
return nil, lastErr
}
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
lastErr = fmt.Errorf("download server error: status=%d body=%s", resp.StatusCode, truncate(string(body), 500))
if attempt < feishuMax5xxRetries {
if sErr := sleepCtx(ctx, feishuRetry5xxDelay); sErr != nil {
return nil, sErr
}
continue
}
return nil, lastErr
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
}
data, readErr := io.ReadAll(resp.Body)
resp.Body.Close()
if readErr != nil {
lastErr = fmt.Errorf("read download body: %w", readErr)
if attempt < feishuMaxRetries {
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
return nil, sErr
}
continue
}
return nil, lastErr
}
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
return data, nil
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read download body: %w", err)
}
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
return data, nil
return nil, lastErr
}
@@ -0,0 +1,173 @@
package feishu
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
)
// retryTestServer builds a server that always answers the auth-token call and
// routes the given target path to h, so tests can drive doRequest's retry loop.
func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) {
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: "fake-token",
Expire: 7200,
})
})
mux.HandleFunc(target, h)
ts := httptest.NewServer(mux)
return ts, &Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}
}
func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
// "0" is coerced to a short delay inside the client so the test stays fast.
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
return
}
writeJSON(w, apiResponse{Code: 0})
})
defer ts.Close()
c := NewClient(cfg)
var resp apiResponse
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil {
t.Fatalf("expected success after retry, got %v", err)
}
if attempts < 2 {
t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts)
}
}
func TestDoRequest_429ExhaustsRetries(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
})
defer ts.Close()
c := NewClient(cfg)
err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil)
if err == nil {
t.Fatal("expected error when 429s exceed the retry budget")
}
if attempts != 4 { // initial + 3 retries
t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts)
}
}
func TestDoRequest_5xxRetriesOnce(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`)
})
defer ts.Close()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
c := NewClient(cfg)
if err := c.doRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil {
t.Fatal("expected error after 5xx exhaustion")
}
if attempts != 2 { // initial + 1 retry
t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts)
}
}
func TestDoRequest_4xxNotRetried(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusBadRequest)
_, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`)
})
defer ts.Close()
c := NewClient(cfg)
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil {
t.Fatal("expected error on 400")
}
if attempts != 1 {
t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts)
}
}
func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("payload-bytes"))
})
defer ts.Close()
c := NewClient(cfg)
data, err := c.downloadRawBytes(context.Background(), "/dl")
if err != nil {
t.Fatalf("expected success after retry, got %v", err)
}
if string(data) != "payload-bytes" {
t.Errorf("data = %q, want %q", string(data), "payload-bytes")
}
if attempts < 2 {
t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts)
}
}
func TestDownloadRawBytes_4xxNotRetried(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusForbidden)
})
defer ts.Close()
c := NewClient(cfg)
if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil {
t.Fatal("expected error on 403")
}
if attempts != 1 {
t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts)
}
}
func TestParseRetryAfter(t *testing.T) {
fallback := 5 * time.Second
tests := []struct {
header string
want time.Duration
}{
{"", fallback},
{"0", 100 * time.Millisecond},
{"-1", 100 * time.Millisecond}, // negative coerced to a short delay
{"3", 3 * time.Second},
{"abc", fallback}, // unparseable
}
for _, tt := range tests {
if got := parseRetryAfter(tt.header, fallback); got != tt.want {
t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want)
}
}
}
+308 -11
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"time"
@@ -27,6 +28,10 @@ func NewConnector(region Region) *Connector {
return &Connector{region: region}
}
// Feishu supports resumable streaming sync; the service prefers FetchStream over
// FetchAll/FetchIncremental when a connector implements StreamingConnector.
var _ datasource.StreamingConnector = (*Connector)(nil)
// Type returns the connector type identifier.
func (c *Connector) Type() string {
return c.region.ConnectorType
@@ -34,6 +39,51 @@ func (c *Connector) Type() string {
const feishuWikiNodeResourceSeparator = ":"
// feishuStreamCheckpointInterval is how many processed nodes pass between
// cursor checkpoints during a streaming fetch. Small enough that a timed-out
// sync loses little work on resume, large enough that checkpoint persistence
// (a DB write) does not dominate. Overridable in tests. See FetchStream.
var feishuStreamCheckpointInterval = 50
// feishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as
// well as node count. Without it, a sync of fewer than
// feishuStreamCheckpointInterval very slow (rate-limited) exports could reach
// the 2h task timeout having never checkpointed, and resume from scratch every
// retry — the #2136 "never fully syncs" case. Overridable in tests.
var feishuStreamCheckpointMaxInterval = 30 * time.Second
// fetchTally accumulates the outcome of fetching a wiki node subtree so the
// connector can emit a single actionable summary. Without it, unsupported nodes
// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users
// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136).
type fetchTally struct {
discovered int
fetched int
failed int
skippedByType map[string]int
}
func newFetchTally(discovered int) *fetchTally {
return &fetchTally{discovered: discovered, skippedByType: map[string]int{}}
}
func (t *fetchTally) fetch() { t.fetched++ }
func (t *fetchTally) fail() { t.failed++ }
func (t *fetchTally) skip(objType string) { t.skippedByType[objType]++ }
func (t *fetchTally) skipped() int {
n := 0
for _, c := range t.skippedByType {
n += c
}
return n
}
func (t *fetchTally) summary() string {
return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v",
t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType)
}
// Validate verifies that the Feishu configuration is valid by testing connectivity.
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
feishuConfig, err := parseFeishuConfig(config, c.region)
@@ -184,25 +234,35 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
allItems = appendWikiNodeListFailureItems(allItems, spaceID, resourceID, partialErr.Failures)
}
// Fetch content for each document node
for _, node := range nodes {
// Fetch content for each document node, tallying outcomes so a single
// summary line explains where every discovered node went.
tally := newFetchTally(len(nodes))
for i, node := range nodes {
item, err := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
if err != nil {
tally.fail()
// Log error but continue with other nodes
allItems = append(allItems, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: resourceID,
Metadata: map[string]string{
"error": err.Error(),
},
Metadata: feishuErrorItemMeta(err, nil),
})
continue
}
if item != nil {
tally.fetch()
allItems = append(allItems, *item)
} else {
// Unsupported obj_type (mindnote/slides/…): skipped with no item.
tally.skip(node.ObjType)
}
if n := i + 1; n%100 == 0 {
logger.Infof(ctx, "[Feishu] sync progress resource=%s %d/%d (%s)",
resourceID, n, len(nodes), tally.summary())
}
}
logger.Infof(ctx, "[Feishu] sync summary resource=%s %s", resourceID, tally.summary())
}
return allItems, nil
@@ -293,9 +353,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: resourceID,
Metadata: map[string]string{
"error": err.Error(),
},
Metadata: feishuErrorItemMeta(err, nil),
})
continue
}
@@ -334,6 +392,246 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
return changedItems, nextSyncCursor, nil
}
// FetchStream performs a resumable, memory-bounded sync. It unifies the full
// and incremental paths: with cursor == nil it fetches everything, and with a
// cursor it skips nodes whose recorded edit time is unchanged — the same
// mechanism that lets a sync which timed out mid-traversal resume from the last
// checkpoint instead of restarting (Tencent/WeKnora#2136).
//
// Instead of accumulating every item in memory (FetchAll), it Emits each item
// as it is fetched and Checkpoints the cursor every feishuStreamCheckpointInterval
// processed nodes, so progress is durable across the Asynq task's 2h timeout.
func (c *Connector) FetchStream(
ctx context.Context, config *types.DataSourceConfig,
cursor *types.SyncCursor, h datasource.StreamHandler,
) (*types.SyncCursor, error) {
feishuConfig, err := parseFeishuConfig(config, c.region)
if err != nil {
return nil, err
}
client := NewClient(feishuConfig)
var prevCursor feishuCursor
if cursor != nil && cursor.ConnectorCursor != nil {
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(cursorBytes, &prevCursor)
}
newCursor := feishuCursor{
LastSyncTime: time.Now(),
SpaceNodeTimes: make(map[string]map[string]string),
}
resourceIDs := config.ResourceIDs
if len(resourceIDs) == 0 {
return nil, fmt.Errorf("no resource IDs (wiki space IDs or wiki node IDs) configured")
}
processed := 0
lastCheckpoint := time.Now()
for _, resourceID := range resourceIDs {
spaceID, nodeToken := parseWikiResourceID(resourceID)
nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken)
var partialErr *partialWikiNodeListError
if err != nil {
if !errors.As(err, &partialErr) {
return nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err)
}
for _, item := range appendWikiNodeListFailureItems(nil, spaceID, resourceID, partialErr.Failures) {
if eerr := h.Emit(ctx, item); eerr != nil {
return nil, eerr
}
}
}
newCursor.SpaceNodeTimes[resourceID] = make(map[string]string)
// On a partial listing, carry prior edit times forward so a later full
// listing can still detect changes and deletions.
if partialErr != nil && prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
for tok, et := range prevTimes {
newCursor.SpaceNodeTimes[resourceID][tok] = et
}
}
}
currentNodes := make(map[string]bool)
tally := newFetchTally(len(nodes))
for i, node := range nodes {
currentNodes[node.NodeToken] = true
editTimeStr := node.ObjEditTime
if editTimeStr == "" {
editTimeStr = node.NodeEditTime
}
// Prior recorded edit time for this node, if any.
var prevEdit string
var hadPrev bool
if prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
prevEdit, hadPrev = prevTimes[node.NodeToken]
}
}
// Resume/incremental fast-path: a node recorded at its current edit
// time is unchanged (or already synced this run) — keep the record
// and skip re-fetching.
if hadPrev && prevEdit == editTimeStr {
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
continue
}
item, ferr := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
if ferr != nil {
tally.fail()
// Do NOT advance the cursor: the content was never fetched.
// Retain the prior edit time (if any) so prev != current next
// run and the node is retried, instead of being permanently
// skipped on a transient export failure (Tencent/WeKnora#2136).
if hadPrev {
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = prevEdit
}
if eerr := h.Emit(ctx, types.FetchedItem{
ExternalID: node.NodeToken,
Title: node.Title,
SourceResourceID: resourceID,
Metadata: feishuErrorItemMeta(ferr, nil),
}); eerr != nil {
return nil, eerr
}
} else {
// Fetched, or an unsupported obj_type (nothing to fetch): record
// the current edit time so the node is not re-processed next run.
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
if item != nil {
tally.fetch()
if eerr := h.Emit(ctx, *item); eerr != nil {
return nil, eerr
}
} else {
// Unsupported obj_type (mindnote/slides/…): no item.
tally.skip(node.ObjType)
}
}
processed++
if processed%feishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= feishuStreamCheckpointMaxInterval {
if cerr := h.Checkpoint(ctx, newCursor.toSyncCursor()); cerr != nil {
logger.Warnf(ctx, "[Feishu] stream checkpoint failed: %v", cerr)
}
lastCheckpoint = time.Now()
}
if n := i + 1; n%100 == 0 {
logger.Infof(ctx, "[Feishu] stream progress resource=%s %d/%d (%s)",
resourceID, n, len(nodes), tally.summary())
}
}
// Detect deleted nodes (only when the full tree was listed successfully).
if partialErr == nil && prevCursor.SpaceNodeTimes != nil {
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
for tok := range prevTimes {
if !currentNodes[tok] {
if eerr := h.Emit(ctx, types.FetchedItem{
ExternalID: tok,
IsDeleted: true,
SourceResourceID: resourceID,
}); eerr != nil {
return nil, eerr
}
}
}
}
}
logger.Infof(ctx, "[Feishu] stream summary resource=%s %s", resourceID, tally.summary())
}
return newCursor.toSyncCursor(), nil
}
// toSyncCursor converts the connector-specific feishuCursor into the generic
// SyncCursor persisted by the service. It marshals through JSON so the returned
// value is a snapshot, decoupled from later mutation of the connector's maps.
func (fc feishuCursor) toSyncCursor() *types.SyncCursor {
m := make(map[string]interface{})
cursorBytes, _ := json.Marshal(fc)
_ = json.Unmarshal(cursorBytes, &m)
return &types.SyncCursor{
LastSyncTime: fc.LastSyncTime,
ConnectorCursor: m,
}
}
var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`)
// feishuErrorCode extracts the numeric Feishu error code from a raw error string
// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort.
func feishuErrorCode(raw string) string {
if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 {
return m[1]
}
return ""
}
// feishuFailure classifies a raw connector/API error into a stable i18n code
// (mapped to a localized string on the frontend), an optional numeric Feishu
// error code for interpolation, and an English fallback message for clients
// without the i18n key. The raw status/JSON body/log_id is never returned here —
// it stays in the server logs. Dumping it in the UI is the anti-pattern
// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync
// (the cursor is retained); auth/permission errors point at the fix instead.
func feishuFailure(err error) (code, codeValue, fallback string) {
if err == nil {
return "sync_failed", "", "Sync failed; will retry on the next sync"
}
s := strings.ToLower(err.Error())
switch {
case strings.Contains(s, "auth error"),
strings.Contains(s, "invalid access token"),
strings.Contains(s, "permission"),
strings.Contains(s, "forbidden"),
strings.Contains(s, "status=403"):
return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes"
case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"):
return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync"
case strings.Contains(s, "timed out"),
strings.Contains(s, "timeout"),
strings.Contains(s, "deadline exceeded"):
return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"
case strings.Contains(s, "server error"):
return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync"
case strings.Contains(s, "api error"),
strings.Contains(s, "export task failed"),
strings.Contains(s, "download failed"):
if v := feishuErrorCode(err.Error()); v != "" {
return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v)
}
return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync"
default:
return "sync_failed", "", "Sync failed; will retry on the next sync"
}
}
// feishuErrorItemMeta builds the metadata for a failed item: the raw error (for
// server logs) plus the classified i18n code / param / fallback (for a
// localisable SyncItemError in the UI), merged with any caller-supplied extras.
func feishuErrorItemMeta(err error, extra map[string]string) map[string]string {
code, codeValue, fallback := feishuFailure(err)
m := map[string]string{
"error": err.Error(),
"error_reason_code": code,
"error_reason": fallback,
}
if codeValue != "" {
m["error_reason_code_value"] = codeValue
}
for k, v := range extra {
m[k] = v
}
return m
}
func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, resourceID string, failures []wikiNodeListFailure) []types.FetchedItem {
for _, failure := range failures {
node := failure.Node
@@ -345,13 +643,12 @@ func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, r
ExternalID: node.NodeToken,
Title: title,
SourceResourceID: resourceID,
Metadata: map[string]string{
"error": failure.Err.Error(),
Metadata: feishuErrorItemMeta(failure.Err, map[string]string{
"channel": types.ChannelFeishu,
"node_token": node.NodeToken,
"space_id": spaceID,
"failure_stage": "list_children",
},
}),
})
}
return items
@@ -0,0 +1,274 @@
package feishu
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// statefulFeishu is a fake Feishu server whose per-node export outcome can be
// changed between sync passes. A node whose obj_token is in failTokens has its
// export status poll return a failed job, so fetchNodeContent errors for it —
// modelling a transient rate-limit / export failure that later recovers.
type statefulFeishu struct {
mu sync.Mutex
failTokens map[string]bool
}
func (s *statefulFeishu) setFail(tokens ...string) {
s.mu.Lock()
defer s.mu.Unlock()
s.failTokens = map[string]bool{}
for _, t := range tokens {
s.failTokens[t] = true
}
}
func (s *statefulFeishu) shouldFail(token string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.failTokens[token]
}
func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFeishu) {
s := &statefulFeishu{failTokens: map[string]bool{}}
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
})
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiSpaceListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
})
})
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiNodeListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{Items: nodes},
})
})
// Export create: ticket == obj_token so the poll below can key on it.
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
var body struct {
Token string `json:"token"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
writeJSON(w, exportTaskCreateResponse{
apiResponse: apiResponse{Code: 0},
Data: struct{ Ticket string `json:"ticket"` }{Ticket: body.Token},
})
})
// Export status poll: /open-apis/drive/v1/export_tasks/<ticket>?token=<objToken>
mux.HandleFunc("/open-apis/drive/v1/export_tasks/", func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "/file/") && strings.HasSuffix(r.URL.Path, "/download") {
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("fake-docx-content"))
return
}
token := r.URL.Query().Get("token")
status := 0
if s.shouldFail(token) {
status = 3 // failed job
}
resp := exportTaskStatusResponse{apiResponse: apiResponse{Code: 0}}
resp.Data.Result.FileToken = "file-" + token
resp.Data.Result.FileName = "exported.docx"
resp.Data.Result.JobStatus = status
resp.Data.Result.JobErrorMsg = "rate limited"
writeJSON(w, resp)
})
ts := httptest.NewServer(mux)
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}, s
}
// convergenceHandler models the service side across a resumable sync:
// - Emit ingests content items (mirroring the service's ctx check + apply),
// records failure items separately, and can simulate a mid-tree task timeout
// by canceling the run's context after cancelAfterCalls Emit calls.
// - Checkpoint snapshots the connector cursor (the DB round-trip).
type convergenceHandler struct {
ingested []string // ExternalIDs of successfully ingested content items
failed []string // ExternalIDs surfaced as failures (Metadata["error"])
exportedTokens map[string]int
checkpoints []feishuCursor
calls int
cancelAfterCall int // >0: cancel ctx and abort on the Nth Emit call
cancel context.CancelFunc
}
func (h *convergenceHandler) Emit(ctx context.Context, item types.FetchedItem) error {
if err := ctx.Err(); err != nil { // mirror service Emit
return err
}
h.calls++
if h.cancelAfterCall > 0 && h.calls >= h.cancelAfterCall {
if h.cancel != nil {
h.cancel()
}
return context.Canceled // service returns ctx err → connector aborts
}
if item.Metadata["error"] != "" {
h.failed = append(h.failed, item.ExternalID)
return nil
}
h.ingested = append(h.ingested, item.ExternalID)
return nil
}
func (h *convergenceHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
var fc feishuCursor
b, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
h.checkpoints = append(h.checkpoints, fc)
return nil
}
func lastCheckpointCursor(h *convergenceHandler, t *testing.T) *types.SyncCursor {
t.Helper()
if len(h.checkpoints) == 0 {
return nil
}
last := h.checkpoints[len(h.checkpoints)-1]
b, _ := json.Marshal(last)
var m map[string]interface{}
if err := json.Unmarshal(b, &m); err != nil {
t.Fatalf("rebuild persisted cursor: %v", err)
}
return &types.SyncCursor{ConnectorCursor: m}
}
// TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure is the
// end-to-end proof for Tencent/WeKnora#2136: a large-ish wiki that (a) hits a
// transient per-node export failure and (b) is killed by the 2h task timeout
// mid-traversal must, on the asynq retry, resume from the last checkpoint,
// re-fetch only what is outstanding, retry the transiently-failed node, and end
// with EVERY document synced exactly once — no permanent skip, no full restart,
// no redundant re-export of already-done nodes.
func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T) {
// Checkpoint on every processed node so the persisted cursor is precise.
prevN := feishuStreamCheckpointInterval
feishuStreamCheckpointInterval = 1
defer func() { feishuStreamCheckpointInterval = prevN }()
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", ObjEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", ObjEditTime: "200"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "docx", Title: "Doc4", ObjEditTime: "400"},
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "docx", Title: "Doc5", ObjEditTime: "500"},
}
ts, cfg, srv := newStatefulFeishu(nodes)
defer ts.Close()
cfgDS := makeConfig(cfg, []string{"space1"})
c := NewConnector(RegionFeishu)
// ---- Pass 1: obj2 export fails transiently; task "times out" on the 3rd
// Emit call (nt1 success, nt2 failure-item, then cancel as nt3 is emitted).
srv.setFail("obj2")
ctx1, cancel1 := context.WithCancel(context.Background())
h1 := &convergenceHandler{cancelAfterCall: 3, cancel: cancel1}
_, err1 := c.FetchStream(ctx1, cfgDS, nil, h1)
cancel1()
if err1 == nil {
t.Fatalf("pass 1 expected a timeout/abort error, got nil")
}
// Pass 1 ingested only nt1; nt2 surfaced as a failure; nt3+ never reached.
if got := strings.Join(h1.ingested, ","); got != "nt1" {
t.Fatalf("pass 1 ingested = %q, want just nt1", got)
}
// The persisted cursor (last checkpoint) must contain nt1 but NOT the
// aborted nt3, nor the transiently-failed nt2 — otherwise resume would skip
// them forever.
persisted := lastCheckpointCursor(h1, t)
if persisted == nil {
t.Fatalf("pass 1 wrote no checkpoint — resume would restart from scratch")
}
var pc feishuCursor
pb, _ := json.Marshal(persisted.ConnectorCursor)
_ = json.Unmarshal(pb, &pc)
p := pc.SpaceNodeTimes["space1"]
if _, ok := p["nt1"]; !ok {
t.Errorf("persisted cursor missing nt1 (done work lost on resume)")
}
if _, ok := p["nt2"]; ok {
t.Errorf("persisted cursor recorded transiently-failed nt2 — it would be skipped forever")
}
if _, ok := p["nt3"]; ok {
t.Errorf("persisted cursor recorded aborted nt3 — it would be skipped forever")
}
// ---- Pass 2: asynq retry. obj2 has recovered; run to completion resuming
// from the persisted cursor.
srv.setFail() // clear all failures
h2 := &convergenceHandler{}
next2, err2 := c.FetchStream(context.Background(), cfgDS, persisted, h2)
if err2 != nil {
t.Fatalf("pass 2 error: %v", err2)
}
// nt1 was already done → must be skipped (not re-exported, not re-ingested).
for _, id := range h2.ingested {
if id == "nt1" {
t.Errorf("pass 2 re-ingested nt1 — redundant re-export of completed work")
}
}
// Everything still outstanding (nt2 transiently-failed, nt3 aborted, nt4, nt5)
// must be ingested in pass 2.
want := map[string]bool{"nt2": true, "nt3": true, "nt4": true, "nt5": true}
got2 := map[string]bool{}
for _, id := range h2.ingested {
got2[id] = true
}
for id := range want {
if !got2[id] {
t.Errorf("pass 2 did not ingest %s (want all outstanding nodes)", id)
}
}
// ---- Convergence: union across both passes covers ALL five nodes exactly.
union := map[string]bool{}
for _, id := range h1.ingested {
union[id] = true
}
for _, id := range h2.ingested {
union[id] = true
}
for _, n := range nodes {
if !union[n.NodeToken] {
t.Errorf("node %s was NEVER synced across both passes — data loss (#2136)", n.NodeToken)
}
}
if len(union) != len(nodes) {
t.Errorf("synced %d distinct nodes, want %d", len(union), len(nodes))
}
// Final cursor is a complete snapshot of every node (next incremental sync
// starts clean).
var fc feishuCursor
nb, _ := json.Marshal(next2.ConnectorCursor)
_ = json.Unmarshal(nb, &fc)
for _, n := range nodes {
if _, ok := fc.SpaceNodeTimes["space1"][n.NodeToken]; !ok {
t.Errorf("final cursor missing %s — incremental sync would re-fetch it", n.NodeToken)
}
}
}
@@ -0,0 +1,108 @@
package feishu
import (
"errors"
"strings"
"testing"
)
// feishuFailure classifies a raw connector/API error into a stable i18n code
// (+ optional numeric feishu code) and an English fallback message. The frontend
// localises by code; the raw status/body/log_id stays in the server logs and
// must never appear in the code, codeValue or fallback.
func TestFeishuFailure(t *testing.T) {
cases := []struct {
name string
err error
wantCode string
wantCodeValue string
fallbackHas []string
noLeak []string // must appear in neither code nor fallback
}{
{
name: "rate limited",
err: errors.New("feishu rate limited: status=429 body={\"code\":99991400,\"msg\":\"too many request\"}"),
wantCode: "feishu_rate_limited",
fallbackHas: []string{"retry"},
noLeak: []string{"body=", "{", "99991400"},
},
{
name: "server 5xx",
err: errors.New("feishu server error: status=500 body={\"code\":1663}"),
wantCode: "feishu_server_unavailable",
noLeak: []string{"body=", "{"},
},
{
name: "export timeout",
err: errors.New("export 季度报告 (docx): export task timed out after 60s (ticket=abc)"),
wantCode: "feishu_timeout",
noLeak: []string{"ticket=", "abc"},
},
{
name: "api error carries the feishu code as a param",
err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"error\":{\"log_id\":\"20260\"}}"),
wantCode: "feishu_api_error",
wantCodeValue: "1663",
noLeak: []string{"log_id", "body=", "{"},
},
{
name: "api error without a code is generic",
err: errors.New("feishu api error: status=502 body=bad gateway"),
wantCode: "feishu_api_error_generic",
},
{
name: "auth failure is actionable, not a retry",
err: errors.New("feishu auth error: code=99991663 msg=Invalid access token"),
wantCode: "feishu_auth_or_permission",
noLeak: []string{"Invalid access token", "99991663"},
},
{
name: "unknown error falls back to sync_failed",
err: errors.New("some totally unexpected failure"),
wantCode: "sync_failed",
noLeak: []string{"totally unexpected"},
},
{
// "decode"/"encode"/"unicode" all contain the substring "code" but
// are not Feishu API errors — they must not be classified as one.
name: "decode error is not a feishu api error",
err: errors.New("failed to decode response body"),
wantCode: "sync_failed",
},
{
// The failed item is retried on the next sync (the cursor is not
// advanced), which for a manual-only source is not automatic. The
// message must not promise an unconditional automatic retry.
name: "transient message promises retry on next sync, not automatic",
err: errors.New("feishu rate limited: status=429"),
wantCode: "feishu_rate_limited",
fallbackHas: []string{"next sync"},
noLeak: []string{"automatically"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
code, codeValue, fallback := feishuFailure(tc.err)
if code != tc.wantCode {
t.Errorf("code = %q, want %q", code, tc.wantCode)
}
if codeValue != tc.wantCodeValue {
t.Errorf("codeValue = %q, want %q", codeValue, tc.wantCodeValue)
}
if strings.TrimSpace(fallback) == "" {
t.Errorf("fallback message must not be empty")
}
for _, s := range tc.fallbackHas {
if !strings.Contains(strings.ToLower(fallback), s) {
t.Errorf("fallback %q missing %q", fallback, s)
}
}
for _, s := range tc.noLeak {
if strings.Contains(code, s) || strings.Contains(fallback, s) {
t.Errorf("raw detail %q leaked into code/fallback (code=%q fallback=%q)", s, code, fallback)
}
}
})
}
}
@@ -0,0 +1,223 @@
//go:build feishu_integration
// Real-API integration test for FetchStream against a live Feishu tenant.
//
// It is OPT-IN and never runs in the normal suite or CI: it is guarded by the
// `feishu_integration` build tag AND skips unless the required env vars are set.
//
// Run it with:
//
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... \
// go test -tags feishu_integration -run TestRealAPI -v \
// ./internal/datasource/connector/feishu/
//
// The test is strictly READ-ONLY: it lists and exports documents from the given
// wiki space but never creates, edits or deletes anything. It whitelists
// *.feishu.cn / *.larksuite.com for SSRF in-process so it works from a dev
// machine behind a fake-ip proxy without any production code change.
package feishu
import (
"context"
"encoding/json"
"os"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
)
// collectHandler records everything a real FetchStream emits so the test can
// assert coverage and resume behaviour. cancelAfter>0 simulates a task timeout
// by cancelling the run after that many successful content emits.
type collectHandler struct {
ingested []string
failed []string
checkpoints []*types.SyncCursor
calls int
cancelAfter int
cancel context.CancelFunc
}
func (h *collectHandler) Emit(ctx context.Context, item types.FetchedItem) error {
if err := ctx.Err(); err != nil {
return err
}
if item.Metadata["error"] != "" {
h.failed = append(h.failed, item.ExternalID)
return nil
}
h.calls++
if h.cancelAfter > 0 && h.calls > h.cancelAfter {
if h.cancel != nil {
h.cancel()
}
return context.Canceled
}
h.ingested = append(h.ingested, item.ExternalID)
return nil
}
func (h *collectHandler) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
// Snapshot the cursor JSON (mirrors the service serialising synchronously).
b, _ := json.Marshal(cursor.ConnectorCursor)
var m map[string]interface{}
_ = json.Unmarshal(b, &m)
h.checkpoints = append(h.checkpoints, &types.SyncCursor{ConnectorCursor: m})
return nil
}
func nodeTimes(t *testing.T, cur *types.SyncCursor) map[string]map[string]string {
t.Helper()
if cur == nil {
return nil
}
var fc feishuCursor
b, _ := json.Marshal(cur.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
return fc.SpaceNodeTimes
}
func countNodes(m map[string]map[string]string) int {
n := 0
for _, v := range m {
n += len(v)
}
return n
}
// TestRealAPI_ListSpaces discovers which wiki spaces the app can access, so the
// space ID for the other tests can be found without guessing. Read-only.
func TestRealAPI_ListSpaces(t *testing.T) {
appID := os.Getenv("FEISHU_APP_ID")
appSecret := os.Getenv("FEISHU_APP_SECRET")
if appID == "" || appSecret == "" {
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
}
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
spaces, err := client.ListWikiSpaces(context.Background())
if err != nil {
t.Fatalf("ListWikiSpaces: %v", err)
}
if len(spaces) == 0 {
t.Fatalf("app can access 0 wiki spaces — add the app as a member of a knowledge base")
}
for _, s := range spaces {
t.Logf("SPACE space_id=%s name=%q", s.SpaceID, s.Name)
}
}
func TestRealAPI_FetchStreamResumeConverges(t *testing.T) {
appID := os.Getenv("FEISHU_APP_ID")
appSecret := os.Getenv("FEISHU_APP_SECRET")
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
if appID == "" || appSecret == "" || spaceID == "" {
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
}
// Whitelist the Feishu/Lark hosts for SSRF so the dev-machine fake-ip proxy
// (198.18.x) is allowed. No production code change; scoped to these hosts.
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
baseURL := os.Getenv("FEISHU_BASE_URL") // optional; defaults to region base
cfg := &types.DataSourceConfig{
Type: types.ConnectorTypeFeishu,
Credentials: map[string]interface{}{
"app_id": appID,
"app_secret": appSecret,
"base_url": baseURL,
},
ResourceIDs: []string{spaceID},
}
c := NewConnector(RegionFeishu)
// ---- Pass 1: full sync against the real space.
h1 := &collectHandler{}
cur1, err := c.FetchStream(context.Background(), cfg, nil, h1)
if err != nil {
t.Fatalf("pass 1 (full) FetchStream error: %v", err)
}
n := len(h1.ingested)
t.Logf("pass 1: ingested=%d failed=%d cursor_nodes=%d", n, len(h1.failed), countNodes(nodeTimes(t, cur1)))
if n == 0 {
t.Fatalf("pass 1 ingested 0 documents — create a few docs in the space, or check app scopes (wiki/docx/drive read)")
}
// Every ingested node must be recorded in the cursor.
nt1 := nodeTimes(t, cur1)[spaceID]
for _, id := range h1.ingested {
if _, ok := nt1[id]; !ok {
t.Errorf("pass 1 cursor missing edit time for ingested node %s", id)
}
}
// ---- Pass 2: incremental with pass-1 cursor. Against the REAL API this is
// the load-bearing check a fake can't make: real obj_edit_time must be
// stable for unchanged docs, so NOTHING is re-ingested.
h2 := &collectHandler{}
cur2, err := c.FetchStream(context.Background(), cfg, cur1, h2)
if err != nil {
t.Fatalf("pass 2 (incremental) FetchStream error: %v", err)
}
t.Logf("pass 2: re-ingested=%d (want 0)", len(h2.ingested))
if len(h2.ingested) != 0 {
t.Errorf("pass 2 re-ingested %v — real edit-time is unstable or skip logic is broken", h2.ingested)
}
if got := countNodes(nodeTimes(t, cur2)); got != countNodes(nt1AsMap(nt1)) {
t.Logf("note: cursor node count changed between passes (%d→%d) — space may have been edited concurrently", countNodes(nt1AsMap(nt1)), got)
}
// ---- Pass 3: interrupt mid-sync then resume, against the real API.
// Only meaningful with at least 2 documents.
if n < 2 {
t.Logf("only %d doc(s); skipping interrupt/resume convergence sub-test (needs >=2)", n)
return
}
prevN := feishuStreamCheckpointInterval
feishuStreamCheckpointInterval = 1 // checkpoint every node so resume is precise
defer func() { feishuStreamCheckpointInterval = prevN }()
ctx3, cancel3 := context.WithCancel(context.Background())
h3 := &collectHandler{cancelAfter: 1, cancel: cancel3} // abort after 1 success
_, err = c.FetchStream(ctx3, cfg, nil, h3)
cancel3()
if err == nil {
t.Fatalf("pass 3 expected an abort error from the simulated timeout")
}
if len(h3.checkpoints) == 0 {
t.Fatalf("pass 3 wrote no checkpoint — resume would restart from scratch")
}
persisted := h3.checkpoints[len(h3.checkpoints)-1]
t.Logf("pass 3: ingested=%d before abort, persisted cursor nodes=%d", len(h3.ingested), countNodes(nodeTimes(t, persisted)))
// Resume from the persisted checkpoint; must converge to full coverage.
h4 := &collectHandler{}
_, err = c.FetchStream(context.Background(), cfg, persisted, h4)
if err != nil {
t.Fatalf("pass 4 (resume) FetchStream error: %v", err)
}
union := map[string]bool{}
for _, id := range h3.ingested {
union[id] = true
}
for _, id := range h4.ingested {
union[id] = true
}
t.Logf("pass 3+4 union ingested=%d, pass-1 full=%d", len(union), n)
if len(union) < n {
t.Errorf("resume converged to %d docs but full sync had %d — a document was lost across interrupt+resume", len(union), n)
}
// Resume must not redo already-checkpointed work.
done := nodeTimes(t, persisted)[spaceID]
for _, id := range h4.ingested {
if _, ok := done[id]; ok {
t.Errorf("pass 4 re-ingested already-checkpointed node %s (redundant re-export)", id)
}
}
}
func nt1AsMap(m map[string]string) map[string]map[string]string {
return map[string]map[string]string{"_": m}
}
@@ -0,0 +1,150 @@
//go:build feishu_integration
// Seeder for the real-API integration test. OPT-IN (feishu_integration tag +
// env vars) so it never runs in the normal suite or CI.
//
// It WRITES to your Feishu tenant: it creates a handful of docx documents under
// the given wiki space, each titled "WeKnora sync test doc N" so you can find
// and delete them afterwards. Requires app write scopes:
// - wiki:wiki (create wiki nodes)
// - docx:document (write docx content)
// plus the app being a member of the space.
//
// Run:
//
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... FEISHU_SEED_COUNT=3 \
// go test -tags feishu_integration -run TestRealAPI_SeedDocs -v \
// ./internal/datasource/connector/feishu/
package feishu
import (
"context"
"fmt"
"os"
"strconv"
"testing"
"github.com/Tencent/WeKnora/internal/utils"
)
// wikiNodeCreateResp is the subset of POST /wiki/v2/spaces/:id/nodes we need.
type wikiNodeCreateResp struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data struct {
Node struct {
NodeToken string `json:"node_token"`
ObjToken string `json:"obj_token"`
ObjType string `json:"obj_type"`
Title string `json:"title"`
} `json:"node"`
} `json:"data"`
}
// TestRealAPI_CreateSpace creates a fresh knowledge base owned by the app, so
// the app can seed and read it without a human adding it to an existing space.
// It prints the new space_id for use as FEISHU_TEST_SPACE_ID.
func TestRealAPI_CreateSpace(t *testing.T) {
appID := os.Getenv("FEISHU_APP_ID")
appSecret := os.Getenv("FEISHU_APP_SECRET")
if appID == "" || appSecret == "" {
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
}
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
var resp struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data struct {
Space struct {
SpaceID string `json:"space_id"`
Name string `json:"name"`
} `json:"space"`
} `json:"data"`
}
err := client.doRequest(context.Background(), "POST", "/open-apis/wiki/v2/spaces",
map[string]interface{}{
"name": "WeKnora Resilience Test KB",
"description": "Auto-created for connector resilience testing (#2136). Safe to delete.",
}, &resp)
if err != nil {
t.Fatalf("create space: %v", err)
}
if resp.Code != 0 {
t.Fatalf("create space: code=%d msg=%s (needs wiki:wiki write scope)", resp.Code, resp.Msg)
}
t.Logf("CREATED_SPACE space_id=%s name=%q", resp.Data.Space.SpaceID, resp.Data.Space.Name)
}
func TestRealAPI_SeedDocs(t *testing.T) {
appID := os.Getenv("FEISHU_APP_ID")
appSecret := os.Getenv("FEISHU_APP_SECRET")
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
if appID == "" || appSecret == "" || spaceID == "" {
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
}
count := 3
if v := os.Getenv("FEISHU_SEED_COUNT"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
count = n
}
}
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
baseURL := os.Getenv("FEISHU_BASE_URL")
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: baseURL})
ctx := context.Background()
for i := 1; i <= count; i++ {
title := fmt.Sprintf("WeKnora sync test doc %d", i)
// 1) Create a docx wiki node.
var created wikiNodeCreateResp
err := client.doRequest(ctx, "POST",
fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes", spaceID),
map[string]interface{}{
"obj_type": "docx",
"node_type": "origin",
"title": title,
}, &created)
if err != nil {
t.Fatalf("create wiki node %d: %v", i, err)
}
if created.Code != 0 {
t.Fatalf("create wiki node %d: code=%d msg=%s (check wiki:wiki write scope + space membership)", i, created.Code, created.Msg)
}
docID := created.Data.Node.ObjToken // for docx, obj_token == document_id
// 2) Add a text block so the export is non-empty.
var blockResp struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
berr := client.doRequest(ctx, "POST",
fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s/children", docID, docID),
map[string]interface{}{
"index": 0,
"children": []map[string]interface{}{{
"block_type": 2, // text
"text": map[string]interface{}{
"elements": []map[string]interface{}{{
"text_run": map[string]interface{}{
"content": fmt.Sprintf("This is WeKnora sync test document number %d. It exists to validate connector resilience (issue #2136).", i),
},
}},
},
}},
}, &blockResp)
if berr != nil {
t.Logf("warn: add content to doc %d failed (%v) — doc still created, export may be empty", i, berr)
} else if blockResp.Code != 0 {
t.Logf("warn: add content to doc %d: code=%d msg=%s (needs docx:document write scope) — doc still created", i, blockResp.Code, blockResp.Msg)
}
t.Logf("created doc %d: title=%q node_token=%s obj_token=%s", i, title, created.Data.Node.NodeToken, docID)
}
t.Logf("seeded %d docs under space %s. Now run TestRealAPI_FetchStreamResumeConverges.", count, spaceID)
}
@@ -0,0 +1,300 @@
package feishu
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// fakeFeishuFailingExport serves the auth/spaces/nodes endpoints normally but
// fails every document export (code != 0), so fetchNodeContent returns an error
// for each supported node — modelling a rate-limited / broken export.
func fakeFeishuFailingExport(nodes []wikiNode) (*httptest.Server, *Config) {
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
})
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiSpaceListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiSpace `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
})
})
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, wikiNodeListResponse{
apiResponse: apiResponse{Code: 0},
Data: struct {
Items []wikiNode `json:"items"`
HasMore bool `json:"has_more"`
PageToken string `json:"page_token"`
}{Items: nodes},
})
})
// Export creation fails for every document.
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, apiResponse{Code: 1, Msg: "export unavailable"})
})
ts := httptest.NewServer(mux)
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}
}
// A node whose fetch fails must NOT have its new edit time recorded in the
// returned cursor: recording it would make the next sync's unchanged fast-path
// skip it forever, silently dropping a document on a transient export failure
// (Tencent/WeKnora#2136). With a prior edit time known, the prior value is
// retained so prev != current next run and the node is retried.
func TestFetchStream_FailedFetchRetainsPriorCursor(t *testing.T) {
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
ts, cfg := fakeFeishuFailingExport(nodes)
defer ts.Close()
cursor := makeStreamCursor(t, map[string]map[string]string{"space1": {"nt1": "50"}}) // prior, older
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h)
if err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
// A failure item must be surfaced, not silently dropped.
if len(h.emitted) != 1 || h.emitted[0].Metadata["error"] == "" {
t.Fatalf("expected 1 emitted failure item with error metadata, got %+v", h.emitted)
}
var fc feishuCursor
b, _ := json.Marshal(next.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
got := fc.SpaceNodeTimes["space1"]["nt1"]
if got == "100" {
t.Fatalf("failed node advanced to current edit time %q — it will be skipped forever", got)
}
if got != "50" {
t.Errorf("failed node cursor = %q, want prior value \"50\" (retry next run)", got)
}
}
// With no prior cursor entry, a failed fetch must leave the node out of the
// returned cursor entirely, so the next run treats it as new and retries it.
func TestFetchStream_FailedFetchNoPriorOmitsFromCursor(t *testing.T) {
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
ts, cfg := fakeFeishuFailingExport(nodes)
defer ts.Close()
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
if err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
var fc feishuCursor
b, _ := json.Marshal(next.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
if v, ok := fc.SpaceNodeTimes["space1"]["nt1"]; ok {
t.Errorf("failed node recorded in cursor as %q; want absent so it is retried next run", v)
}
}
// recordingHandler captures the items and checkpoints a streaming fetch emits.
// Checkpoints are snapshotted (JSON-encoded) at call time — mirroring the
// service, which serializes the cursor synchronously inside Checkpoint — so the
// test observes the cursor state as it was when Checkpoint was called, not the
// connector's later-mutated map.
type recordingHandler struct {
emitted []types.FetchedItem
checkpoints []feishuCursor
emitErr func(item types.FetchedItem) error
}
func (h *recordingHandler) Emit(ctx context.Context, item types.FetchedItem) error {
if h.emitErr != nil {
if err := h.emitErr(item); err != nil {
return err
}
}
h.emitted = append(h.emitted, item)
return nil
}
func (h *recordingHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
var fc feishuCursor
b, _ := json.Marshal(cursor.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
h.checkpoints = append(h.checkpoints, fc)
return nil
}
func makeStreamCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) *types.SyncCursor {
t.Helper()
prev := feishuCursor{SpaceNodeTimes: spaceNodeTimes}
b, _ := json.Marshal(prev)
var m map[string]interface{}
if err := json.Unmarshal(b, &m); err != nil {
t.Fatalf("build cursor: %v", err)
}
return &types.SyncCursor{ConnectorCursor: m}
}
// A nil-cursor stream emits every supported node once and skips unsupported
// types, and the returned cursor records the edit time of every discovered node
// (so the next incremental run can detect changes).
func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "mindnote", Title: "Brain", ObjEditTime: "200"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
if err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
if len(h.emitted) != 2 {
t.Fatalf("emitted %d items, want 2 (nt1, nt3)", len(h.emitted))
}
if h.emitted[0].ExternalID != "nt1" || h.emitted[1].ExternalID != "nt3" {
t.Errorf("emitted ids = %q,%q; want nt1,nt3", h.emitted[0].ExternalID, h.emitted[1].ExternalID)
}
var fc feishuCursor
b, _ := json.Marshal(next.ConnectorCursor)
_ = json.Unmarshal(b, &fc)
times := fc.SpaceNodeTimes["space1"]
for _, tok := range []string{"nt1", "nt2", "nt3"} {
if _, ok := times[tok]; !ok {
t.Errorf("returned cursor missing edit time for %s", tok)
}
}
}
// When a cursor already records a node at its current edit time, that node is
// unchanged and must not be re-emitted; only new/changed nodes stream through.
// This is the resume/incremental-skip behavior that lets a timed-out sync
// converge across retries instead of re-exporting everything.
func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
cursor := makeStreamCursor(t, map[string]map[string]string{
"space1": {"nt1": "100"}, // nt1 unchanged; nt3 unknown → new
})
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h); err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
if len(h.emitted) != 1 {
t.Fatalf("emitted %d items, want 1 (only changed nt3)", len(h.emitted))
}
if h.emitted[0].ExternalID != "nt3" {
t.Errorf("emitted id = %q, want nt3", h.emitted[0].ExternalID)
}
}
// Checkpoints must persist progress at page boundaries so a crash mid-sync
// resumes from the last checkpoint. With the interval set to 1, each emitted
// item triggers a checkpoint, and the first checkpoint must already contain the
// first node's edit time.
func TestFetchStream_CheckpointsProgress(t *testing.T) {
prev := feishuStreamCheckpointInterval
feishuStreamCheckpointInterval = 1
defer func() { feishuStreamCheckpointInterval = prev }()
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
if len(h.checkpoints) == 0 {
t.Fatalf("expected at least one checkpoint")
}
first := h.checkpoints[0]
if _, ok := first.SpaceNodeTimes["space1"]["nt1"]; !ok {
t.Errorf("first checkpoint missing nt1 progress: %+v", first.SpaceNodeTimes)
}
}
// Checkpoints must ALSO fire on elapsed time, not only every N nodes.
// Otherwise a sync with fewer than the node interval of slow (rate-limited)
// exports reaches the 2h task timeout having never checkpointed, and resumes
// from scratch forever — exactly the #2136 "never fully syncs" case. With the
// node interval effectively disabled and the time interval at 0, every
// processed node must still produce a checkpoint.
func TestFetchStream_CheckpointsOnElapsedTime(t *testing.T) {
prevN := feishuStreamCheckpointInterval
prevT := feishuStreamCheckpointMaxInterval
feishuStreamCheckpointInterval = 1 << 30 // never fires by count
feishuStreamCheckpointMaxInterval = 0 // fires by elapsed time every node
defer func() {
feishuStreamCheckpointInterval = prevN
feishuStreamCheckpointMaxInterval = prevT
}()
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
c := NewConnector(RegionFeishu)
h := &recordingHandler{}
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
t.Fatalf("FetchStream() error: %v", err)
}
if len(h.checkpoints) == 0 {
t.Fatalf("expected time-based checkpoints even though node interval never fires")
}
}
// An Emit error aborts the stream immediately — the connector must return the
// error and stop fetching further nodes (the sync is failing; do not burn API
// budget on the rest of the tree).
func TestFetchStream_EmitErrorAborts(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
boom := errors.New("ingest failed")
c := NewConnector(RegionFeishu)
h := &recordingHandler{emitErr: func(item types.FetchedItem) error { return boom }}
_, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
if !errors.Is(err, boom) {
t.Fatalf("FetchStream() error = %v, want %v", err, boom)
}
if len(h.emitted) != 0 {
t.Errorf("emitted %d items, want 0 (aborted on first emit)", len(h.emitted))
}
}
@@ -1,6 +1,7 @@
package feishu
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -12,8 +13,9 @@ import (
"time"
"unicode/utf8"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
func TestMain(m *testing.M) {
@@ -775,6 +777,39 @@ func TestFetchAll_MixedTypes(t *testing.T) {
}
}
func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
var buf bytes.Buffer
logger.SetOutput(&buf)
defer logger.SetOutput(os.Stderr)
c := NewConnector(RegionFeishu)
if _, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}); err != nil {
t.Fatalf("FetchAll() error: %v", err)
}
out := buf.String()
for _, want := range []string{
"sync summary",
"discovered=3",
"fetched=1",
"skipped_unsupported=2",
"mindnote:1",
"slides:1",
} {
if !strings.Contains(out, want) {
t.Errorf("summary log missing %q; got:\n%s", want, out)
}
}
}
func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) {
nodes := []wikiNode{
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
@@ -0,0 +1,45 @@
package feishu
import (
"strings"
"testing"
)
func TestFetchTally_CountsAndSummary(t *testing.T) {
tally := newFetchTally(13)
tally.fetch()
tally.fetch()
tally.fetch()
tally.skip("mindnote")
tally.skip("mindnote")
tally.skip("slides")
tally.fail()
if got := tally.skipped(); got != 3 {
t.Errorf("skipped() = %d, want 3", got)
}
summary := tally.summary()
for _, want := range []string{
"discovered=13",
"fetched=3",
"failed=1",
"skipped_unsupported=3",
"mindnote:2",
"slides:1",
} {
if !strings.Contains(summary, want) {
t.Errorf("summary() = %q, missing %q", summary, want)
}
}
}
func TestFetchTally_EmptyHasNoSkips(t *testing.T) {
tally := newFetchTally(0)
if got := tally.skipped(); got != 0 {
t.Errorf("skipped() = %d, want 0", got)
}
if !strings.Contains(tally.summary(), "skipped_unsupported=0") {
t.Errorf("summary() = %q, want skipped_unsupported=0", tally.summary())
}
}
+49 -2
View File
@@ -358,13 +358,60 @@ type SyncResult struct {
// Items that failed
Failed int `json:"failed"`
// Detailed error messages
Errors []string `json:"errors,omitempty"`
// Per-item failure samples (capped), shown in the sync-log UI.
Errors []SyncItemError `json:"errors,omitempty"`
// Updated cursor for next incremental sync
NextCursor *SyncCursor `json:"next_cursor,omitempty"`
}
// SyncItemError is one user-facing failure sample. It carries a stable i18n
// Code (+ interpolation Params) so the frontend localises it to the viewer's
// language, plus a Message fallback for clients without the key. The raw API
// status/body/log_id is never stored here — that stays in the server logs.
type SyncItemError struct {
// Title is the document title (user content, not translated).
Title string `json:"title,omitempty"`
// Code is a stable key the frontend maps to a localized string, e.g.
// "feishu_rate_limited" → datasource.syncError.feishu_rate_limited.
Code string `json:"code,omitempty"`
// Params are interpolation values for the localized string, e.g. {"code":"1663"}.
Params map[string]string `json:"params,omitempty"`
// Message is a human fallback used when the client has no i18n key for Code.
Message string `json:"message,omitempty"`
}
// Display renders the sample as a single plain string for server-side use
// (logs, fatal-error detail). The localised UI string is built on the frontend
// from Code/Params; this is only a non-localised fallback.
func (e SyncItemError) Display() string {
switch {
case e.Title != "" && e.Message != "":
return e.Title + ": " + e.Message
case e.Message != "":
return e.Message
default:
return e.Title
}
}
// UnmarshalJSON keeps old sync logs readable: historically each error was a
// plain JSON string, so a bare string decodes into Message.
func (e *SyncItemError) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err == nil {
e.Message = s
return nil
}
type alias SyncItemError
var a alias
if err := json.Unmarshal(b, &a); err != nil {
return err
}
*e = SyncItemError(a)
return nil
}
// DataSourceSyncPayload represents the asynq task payload for data source sync
type DataSourceSyncPayload struct {
TracingContext
+44
View File
@@ -0,0 +1,44 @@
package types
import (
"encoding/json"
"testing"
)
// SyncResult.Errors is persisted as jsonb and read back when the sync-log drawer
// is opened. Historically each entry was a plain string; it is now a structured
// SyncItemError (title + i18n code + params + fallback message) so the frontend
// can localise it. Old rows must still deserialize, so a bare string decodes into
// the Message field.
func TestSyncItemError_UnmarshalAcceptsLegacyStringAndObject(t *testing.T) {
// Legacy format: array of plain strings.
var legacy []SyncItemError
if err := json.Unmarshal([]byte(`["季度报告: export failed"]`), &legacy); err != nil {
t.Fatalf("legacy string form must still decode: %v", err)
}
if len(legacy) != 1 || legacy[0].Message != "季度报告: export failed" {
t.Fatalf("legacy string should map to Message, got %+v", legacy)
}
if legacy[0].Code != "" {
t.Errorf("legacy string must not invent a code, got %q", legacy[0].Code)
}
// New format: structured object round-trips.
in := SyncItemError{
Title: "季度报告",
Code: "feishu_api_error",
Params: map[string]string{"code": "1663"},
Message: "Feishu API error (code=1663); will retry",
}
b, err := json.Marshal([]SyncItemError{in})
if err != nil {
t.Fatalf("marshal: %v", err)
}
var out []SyncItemError
if err := json.Unmarshal(b, &out); err != nil {
t.Fatalf("unmarshal structured: %v", err)
}
if len(out) != 1 || out[0].Code != "feishu_api_error" || out[0].Params["code"] != "1663" || out[0].Title != "季度报告" {
t.Fatalf("structured form did not round-trip: %+v", out)
}
}