fix(feishu): resilient large-wiki sync — retry, streaming resume, failure visibility
Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136: 5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run). Three compounding causes, fixed in layers: 1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export APIs are aggressively rate limited; a thousand-doc sync issues tens of thousands of calls, and a single 429/5xx burst failed whole swathes of docs. Add exponential backoff honouring Retry-After to doRequest (JSON) and downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off. 2. The batch path fetched the entire tree into memory before ingesting, so a sync that hit the 2h task timeout lost all progress and restarted from scratch, and a transient per-document export failure advanced the cursor and skipped that doc permanently. Add an optional StreamingConnector interface, implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint, bounding memory to one item and persisting a per-node edit-time cursor at page boundaries. The service dispatches to it via type assertion; connectors without it fall back to FetchAll / FetchIncremental unchanged (non-breaking). A resumed/incremental run skips nodes already recorded at their current edit time (converges), a failed fetch does not advance the cursor (retried next run instead of dropped), and partial listing failures emit error items and carry prior edit times forward instead of aborting the sync. 3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…) were skipped with no item, error or log ("13 docs synced only 3"). FetchAll now tallies discovered/fetched/failed/skipped per resource with a per-type skip breakdown and a progress line every 100 nodes. The backend already persisted per-item errors in SyncLog.result.errors but the frontend type omitted the field; declare `result` on SyncLog and render up to 50 failed documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU. result.errors is a jsonb column shipped in every list response, so it is capped at 100 samples server-side (recordSyncError); the exact failure total comes from the bounded items_failed counter, which the drawer uses for its count and "N more" line so a sync that fails thousands of docs stays bounded. Each sample is a structured, localisable SyncItemError instead of a raw "title: ...api error: status=500 body={json}" dump that repeated the title and leaked status/body/log_id. The connector (feishuFailure) classifies the error into a stable i18n code (+ the feishu code as a param); the service builds {title, code, params, message}; the drawer localises via vue-i18n to the viewer's language (zh/en/ko/ru), falling back to message. A custom UnmarshalJSON keeps old string-form logs readable. The banner is localised from items_failed too. Raw errors stay in the server logs. Follows the two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised summaries in the UI; raw detail in logs. Transient errors say "will retry" (the cursor is retained); auth/permission errors point at the fix instead. Tests: client_retry_test (backoff), tally_test (accounting), connector_stream / datasource_stream (streaming dispatch + checkpoint), connector_convergence (timeout + transient-failure resume converges to full coverage; proven to catch #2136 via bug injection), datasource_result_cap (error-sample cap + structured localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code classification), sync_item_error (legacy-string + structured back-compat) — all in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu harness gated by the feishu_integration build tag AND env vars, never run in CI.
This commit is contained in:
@@ -25,6 +25,29 @@ export interface DataSource {
|
||||
latest_sync_log?: SyncLog
|
||||
}
|
||||
|
||||
/**
|
||||
* One user-facing failure sample. The backend sends a stable i18n `code`
|
||||
* (+ interpolation `params`) so the UI localises it to the viewer's language;
|
||||
* `message` is a fallback when no i18n key exists (old logs decode into it).
|
||||
*/
|
||||
export interface SyncItemError {
|
||||
title?: string
|
||||
code?: string
|
||||
params?: Record<string, string>
|
||||
message?: string
|
||||
}
|
||||
|
||||
export interface SyncResultDetail {
|
||||
total?: number
|
||||
created?: number
|
||||
updated?: number
|
||||
deleted?: number
|
||||
skipped?: number
|
||||
failed?: number
|
||||
/** Per-item failure samples (capped); localised in the sync-log drawer. */
|
||||
errors?: SyncItemError[]
|
||||
}
|
||||
|
||||
export interface SyncLog {
|
||||
id: string
|
||||
data_source_id: string
|
||||
@@ -38,6 +61,7 @@ export interface SyncLog {
|
||||
items_skipped: number
|
||||
items_failed: number
|
||||
error_message: string
|
||||
result?: SyncResultDetail
|
||||
}
|
||||
|
||||
export interface ConnectorMeta {
|
||||
|
||||
@@ -6216,6 +6216,19 @@ export default {
|
||||
startTime: 'Started',
|
||||
endTime: 'Finished',
|
||||
duration: 'Duration',
|
||||
failedItems: 'Failed documents',
|
||||
failedItemsMore: '{n} more failed documents not shown',
|
||||
docsFailedSummary: '{n} document(s) failed to sync',
|
||||
},
|
||||
syncError: {
|
||||
feishu_auth_or_permission: 'Authentication or permission error; check credentials and app permissions',
|
||||
feishu_rate_limited: 'Feishu API rate limited; will retry on the next sync',
|
||||
feishu_timeout: 'Export or request timed out; will retry on the next sync',
|
||||
feishu_server_unavailable: 'Feishu service temporarily unavailable; will retry on the next sync',
|
||||
feishu_api_error: 'Feishu API error (code={code}); will retry on the next sync',
|
||||
feishu_api_error_generic: 'Feishu API error; will retry on the next sync',
|
||||
sync_failed: 'Sync failed; will retry on the next sync',
|
||||
ingest_failed: 'Ingest failed; see server logs',
|
||||
},
|
||||
connector: {
|
||||
feishu: 'Feishu',
|
||||
|
||||
@@ -6221,6 +6221,19 @@ export default {
|
||||
startTime: "시작 시간",
|
||||
endTime: "종료 시간",
|
||||
duration: "소요 시간",
|
||||
failedItems: "실패한 문서",
|
||||
failedItemsMore: "표시되지 않은 실패 문서 {n}개",
|
||||
docsFailedSummary: "문서 {n}개 동기화 실패",
|
||||
},
|
||||
syncError: {
|
||||
feishu_auth_or_permission: "인증 또는 권한 부족: 자격 증명과 앱 권한을 확인하세요",
|
||||
feishu_rate_limited: "Feishu API 요청 제한, 다음 동기화 시 다시 시도합니다",
|
||||
feishu_timeout: "문서 내보내기 또는 요청 시간 초과, 다음 동기화 시 다시 시도합니다",
|
||||
feishu_server_unavailable: "Feishu 서비스가 일시적으로 사용 불가, 다음 동기화 시 다시 시도합니다",
|
||||
feishu_api_error: "Feishu API 오류(code={code}), 다음 동기화 시 다시 시도합니다",
|
||||
feishu_api_error_generic: "Feishu API 오류, 다음 동기화 시 다시 시도합니다",
|
||||
sync_failed: "동기화 실패, 다음 동기화 시 다시 시도합니다",
|
||||
ingest_failed: "가져오기 실패, 서버 로그를 확인하세요",
|
||||
},
|
||||
connector: {
|
||||
feishu: "페이슈 (Feishu)",
|
||||
|
||||
@@ -6042,6 +6042,19 @@ export default {
|
||||
startTime: 'Время начала',
|
||||
endTime: 'Время окончания',
|
||||
duration: 'Длительность',
|
||||
failedItems: 'Документы с ошибкой',
|
||||
failedItemsMore: 'Ещё {n} документов с ошибкой не показаны',
|
||||
docsFailedSummary: 'Не удалось синхронизировать документов: {n}',
|
||||
},
|
||||
syncError: {
|
||||
feishu_auth_or_permission: 'Ошибка аутентификации или недостаточно прав; проверьте учётные данные и права приложения',
|
||||
feishu_rate_limited: 'Превышен лимит запросов к API Feishu; повтор при следующей синхронизации',
|
||||
feishu_timeout: 'Тайм-аут экспорта или запроса; повтор при следующей синхронизации',
|
||||
feishu_server_unavailable: 'Сервис Feishu временно недоступен; повтор при следующей синхронизации',
|
||||
feishu_api_error: 'Ошибка API Feishu (code={code}); повтор при следующей синхронизации',
|
||||
feishu_api_error_generic: 'Ошибка API Feishu; повтор при следующей синхронизации',
|
||||
sync_failed: 'Ошибка синхронизации; повтор при следующей синхронизации',
|
||||
ingest_failed: 'Ошибка импорта; смотрите логи сервера',
|
||||
},
|
||||
connector: {
|
||||
feishu: 'Feishu (Фэйшу)',
|
||||
|
||||
@@ -6241,6 +6241,19 @@ export default {
|
||||
startTime: "开始时间",
|
||||
endTime: "结束时间",
|
||||
duration: "耗时",
|
||||
failedItems: "失败文档",
|
||||
failedItemsMore: "还有 {n} 个失败文档未显示",
|
||||
docsFailedSummary: "{n} 个文档同步失败",
|
||||
},
|
||||
syncError: {
|
||||
feishu_auth_or_permission: "应用鉴权或权限不足,请检查凭证与应用权限",
|
||||
feishu_rate_limited: "飞书接口限流,下次同步时将重试",
|
||||
feishu_timeout: "文档导出或请求超时,下次同步时将重试",
|
||||
feishu_server_unavailable: "飞书服务暂时不可用,下次同步时将重试",
|
||||
feishu_api_error: "飞书接口错误(code={code}),下次同步时将重试",
|
||||
feishu_api_error_generic: "飞书接口错误,下次同步时将重试",
|
||||
sync_failed: "同步失败,下次同步时将重试",
|
||||
ingest_failed: "入库失败,详情见服务日志",
|
||||
},
|
||||
connector: {
|
||||
feishu: "飞书",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { getSyncLogs, type SyncLog } from '@/api/datasource'
|
||||
import { getSyncLogs, type SyncLog, type SyncItemError } from '@/api/datasource'
|
||||
|
||||
const props = defineProps<{
|
||||
dataSourceId: string
|
||||
@@ -131,6 +131,30 @@ function hasPills(log: SyncLog) {
|
||||
return log.items_created > 0 || log.items_updated > 0 || log.items_deleted > 0 || log.items_skipped > 0 || log.items_failed > 0
|
||||
}
|
||||
|
||||
// Cap the per-item failure list so a sync that failed thousands of documents
|
||||
// doesn't render an unbounded wall of text; the remainder is summarised.
|
||||
const FAILED_ITEMS_CAP = 50
|
||||
|
||||
function failedItems(log: SyncLog): SyncItemError[] {
|
||||
return (log.result?.errors || []).slice(0, FAILED_ITEMS_CAP)
|
||||
}
|
||||
|
||||
// Render one failure sample. The backend sends a stable i18n `code` (+ params)
|
||||
// so the reason is localised to the viewer's language; `message` is the fallback
|
||||
// for old logs / codes this client doesn't know. The document title is kept
|
||||
// separate and prefixed as "title — reason".
|
||||
function formatSyncError(e: SyncItemError): string {
|
||||
let reason = ''
|
||||
if (e.code) {
|
||||
const key = `datasource.syncError.${e.code}`
|
||||
const localised = t(key, (e.params || {}) as Record<string, unknown>)
|
||||
reason = localised === key ? (e.message || e.code) : localised
|
||||
} else {
|
||||
reason = e.message || ''
|
||||
}
|
||||
return e.title ? (reason ? `${e.title} — ${reason}` : e.title) : reason
|
||||
}
|
||||
|
||||
// Group logs by date
|
||||
const groupedLogs = computed(() => {
|
||||
const groups: { date: string; logs: SyncLog[] }[] = []
|
||||
@@ -253,9 +277,29 @@ const groupedLogs = computed(() => {
|
||||
<span class="detail-label">{{ t('datasource.logMetric.total') }}</span>
|
||||
<span>{{ log.items_total }}</span>
|
||||
</div>
|
||||
<div v-if="log.error_message" class="tl-error">
|
||||
<!-- Localised failure summary; raw error_message only for a
|
||||
pure infra failure with no per-document detail. -->
|
||||
<div v-if="log.items_failed > 0" class="tl-error">
|
||||
{{ t('datasource.logDetail.docsFailedSummary', { n: log.items_failed }) }}
|
||||
</div>
|
||||
<div v-else-if="log.error_message" class="tl-error">
|
||||
{{ log.error_message }}
|
||||
</div>
|
||||
|
||||
<!-- Per-item failures: which documents failed and why.
|
||||
The true count is items_failed (a bounded int); result.errors
|
||||
is only a capped sample the backend retains for display. -->
|
||||
<div v-if="failedItems(log).length" class="tl-failed">
|
||||
<div class="tl-failed-title">
|
||||
{{ t('datasource.logDetail.failedItems') }} ({{ log.items_failed }})
|
||||
</div>
|
||||
<div v-for="(e, i) in failedItems(log)" :key="i" class="tl-failed-item" :title="formatSyncError(e)">
|
||||
{{ formatSyncError(e) }}
|
||||
</div>
|
||||
<div v-if="log.items_failed > failedItems(log).length" class="tl-failed-more">
|
||||
{{ t('datasource.logDetail.failedItemsMore', { n: (log.items_failed - failedItems(log).length) }) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -513,6 +557,39 @@ const groupedLogs = computed(() => {
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
/* --- Per-item failure list --- */
|
||||
.tl-failed {
|
||||
margin-top: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.tl-failed-title {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: var(--td-error-color);
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.tl-failed-item {
|
||||
font-size: 11px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-secondary);
|
||||
padding: 2px 8px;
|
||||
border-left: 2px solid var(--td-error-color-3);
|
||||
background: var(--td-bg-color-container);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.tl-failed-more {
|
||||
font-size: 11px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
padding: 2px 8px;
|
||||
}
|
||||
|
||||
/* --- Drawer header --- */
|
||||
.logs-drawer-header {
|
||||
display: flex;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// A sync that fails thousands of documents must not accumulate an unbounded
|
||||
// result.Errors slice: that slice is persisted as jsonb and shipped in every
|
||||
// sync-log list response, so an uncapped list means multi-MB rows and payloads.
|
||||
// The accurate count lives in result.Failed (a bounded int), so Errors only
|
||||
// needs to retain a sample for display (Tencent/WeKnora#2136 / #1262).
|
||||
func TestApplyFetchedItem_CapsErrorSampleButKeepsAccurateFailedCount(t *testing.T) {
|
||||
s := &DataSourceService{}
|
||||
ds := &types.DataSource{}
|
||||
result := &types.SyncResult{}
|
||||
ctx := context.Background()
|
||||
|
||||
const failures = 5000
|
||||
for i := 0; i < failures; i++ {
|
||||
s.applyFetchedItem(ctx, ds, &types.FetchedItem{
|
||||
ExternalID: "node",
|
||||
Title: "doc",
|
||||
Metadata: map[string]string{"error": "export failed: rate limited"},
|
||||
}, nil, result)
|
||||
}
|
||||
|
||||
assert.Equal(t, failures, result.Failed,
|
||||
"Failed must count every failure regardless of the error-sample cap")
|
||||
assert.LessOrEqual(t, len(result.Errors), maxSyncResultErrors,
|
||||
"Errors sample must be capped so the persisted jsonb stays bounded")
|
||||
assert.Equal(t, maxSyncResultErrors, len(result.Errors),
|
||||
"with more failures than the cap, the sample should be exactly the cap")
|
||||
}
|
||||
|
||||
// The user-facing sample must be a structured SyncItemError carrying the
|
||||
// connector's stable i18n code + params (so the frontend localises it), the
|
||||
// title as a separate field, and no raw API body/log_id — that stays in logs.
|
||||
func TestApplyFetchedItem_ProducesLocalisableStructuredError(t *testing.T) {
|
||||
s := &DataSourceService{}
|
||||
ds := &types.DataSource{}
|
||||
result := &types.SyncResult{}
|
||||
|
||||
s.applyFetchedItem(context.Background(), ds, &types.FetchedItem{
|
||||
ExternalID: "nt1",
|
||||
Title: "季度报告",
|
||||
Metadata: map[string]string{
|
||||
// raw, for logs only
|
||||
"error": `export 季度报告 (docx): feishu api error: status=500 body={"code":1663,"error":{"log_id":"20260"}}`,
|
||||
// classification, for a localisable UI sample
|
||||
"error_reason_code": "feishu_api_error",
|
||||
"error_reason_code_value": "1663",
|
||||
"error_reason": "Feishu API error (code=1663); will retry automatically",
|
||||
},
|
||||
}, nil, result)
|
||||
|
||||
assert.Equal(t, 1, len(result.Errors))
|
||||
got := result.Errors[0]
|
||||
assert.Equal(t, "季度报告", got.Title, "title is a separate field, not embedded in the message")
|
||||
assert.Equal(t, "feishu_api_error", got.Code, "stable i18n code drives frontend localisation")
|
||||
assert.Equal(t, "1663", got.Params["code"], "feishu error code is passed as an interpolation param")
|
||||
assert.NotContains(t, got.Message, "body=", "raw API body must not reach the UI sample")
|
||||
assert.NotContains(t, got.Message, "log_id", "raw log_id must not reach the UI sample")
|
||||
}
|
||||
@@ -625,6 +625,13 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
return err
|
||||
}
|
||||
|
||||
// Streaming path: connectors that support it interleave fetch→ingest→
|
||||
// checkpoint so a large sync bounds memory and resumes after a timeout
|
||||
// instead of restarting (Tencent/WeKnora#2136). Others fall back below.
|
||||
if sc, ok := connector.(datasource.StreamingConnector); ok {
|
||||
return s.processSyncStreaming(ctx, sc, ds, syncLog, config, payload, wasPaused)
|
||||
}
|
||||
|
||||
// Fetch items based on sync mode
|
||||
var items []types.FetchedItem
|
||||
var nextCursor *types.SyncCursor
|
||||
@@ -697,56 +704,11 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
||||
|
||||
// Auto-tag: find or create a tag for this data source so synced items are easily identifiable
|
||||
autoTagIDs := []string{}
|
||||
autoTagName := ds.Name
|
||||
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, autoTagName); tagErr != nil {
|
||||
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", autoTagName, tagErr)
|
||||
} else if autoTag != nil {
|
||||
autoTagIDs = append(autoTagIDs, autoTag.ID)
|
||||
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", autoTagName, autoTag.ID)
|
||||
}
|
||||
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
|
||||
|
||||
for _, item := range items {
|
||||
if item.IsDeleted {
|
||||
if ds.SyncDeletions {
|
||||
// Count only — actual KB deletion is intentionally not performed.
|
||||
// Users manage knowledge removal explicitly via the KB UI to avoid
|
||||
// accidental data loss from connector misdetection or reconfiguration.
|
||||
result.Deleted++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if len(item.Content) == 0 && item.URL == "" {
|
||||
// Check if this is an error item from the connector (failed to fetch content)
|
||||
if errMsg, hasErr := item.Metadata["error"]; hasErr {
|
||||
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
|
||||
result.Failed++
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("%s: %s", item.Title, errMsg))
|
||||
} else {
|
||||
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
|
||||
result.Skipped++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
isUpdate, err := s.ingestItem(ctx, ds, &item, autoTagIDs)
|
||||
if err != nil {
|
||||
// Duplicate file/URL is not a failure — count as skipped
|
||||
var dupErr *types.DuplicateKnowledgeError
|
||||
if errors.As(err, &dupErr) {
|
||||
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
|
||||
result.Skipped++
|
||||
} else {
|
||||
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
|
||||
result.Failed++
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", item.Title, err))
|
||||
}
|
||||
} else if isUpdate {
|
||||
result.Updated++
|
||||
} else {
|
||||
result.Created++
|
||||
}
|
||||
item := item
|
||||
s.applyFetchedItem(ctx, ds, &item, autoTagIDs, result)
|
||||
}
|
||||
|
||||
resultJSON, _ := result.ToJSON()
|
||||
@@ -769,7 +731,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
syncStatus = types.SyncLogStatusPartial
|
||||
syncErrorMessage = fmt.Sprintf("Some feeds failed: %s", strings.Join(fetchWarnings, "; "))
|
||||
for _, w := range fetchWarnings {
|
||||
result.Errors = append(result.Errors, w)
|
||||
result.Errors = append(result.Errors, types.SyncItemError{Message: w})
|
||||
}
|
||||
resultJSON, _ = result.ToJSON()
|
||||
}
|
||||
@@ -781,6 +743,253 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveAutoTagIDs finds or creates the per-data-source tag applied to every
|
||||
// synced item so results are identifiable in the KB. A tag failure is
|
||||
// non-fatal: the sync proceeds untagged.
|
||||
func (s *DataSourceService) resolveAutoTagIDs(ctx context.Context, ds *types.DataSource) []string {
|
||||
autoTagIDs := []string{}
|
||||
if autoTag, tagErr := s.tagService.FindOrCreateTagByName(ctx, ds.KnowledgeBaseID, ds.Name); tagErr != nil {
|
||||
logger.Warnf(ctx, "failed to find/create auto-tag %q: %v (proceeding without tag)", ds.Name, tagErr)
|
||||
} else if autoTag != nil {
|
||||
autoTagIDs = append(autoTagIDs, autoTag.ID)
|
||||
logger.Infof(ctx, "using auto-tag %q (id=%s) for data source sync", ds.Name, autoTag.ID)
|
||||
}
|
||||
return autoTagIDs
|
||||
}
|
||||
|
||||
// maxSyncResultErrors bounds the per-item error sample retained in
|
||||
// SyncResult.Errors. That slice is persisted as jsonb and returned in every
|
||||
// sync-log list response, so an unbounded list on a sync that fails thousands of
|
||||
// documents means multi-MB DB rows and payloads. The accurate failure count
|
||||
// lives in SyncResult.Failed (a bounded int); this list only keeps a sample for
|
||||
// display (Tencent/WeKnora#2136 / #1262).
|
||||
const maxSyncResultErrors = 100
|
||||
|
||||
// recordSyncError appends an error sample to result.Errors, capped at
|
||||
// maxSyncResultErrors. Callers still increment result.Failed for the exact count.
|
||||
func recordSyncError(result *types.SyncResult, item types.SyncItemError) {
|
||||
if len(result.Errors) < maxSyncResultErrors {
|
||||
result.Errors = append(result.Errors, item)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchFailureSyncError maps a connector error item into a structured, user-
|
||||
// facing sample. Connectors that classify their errors (Feishu) provide a stable
|
||||
// i18n code + params via metadata so the frontend localises it to the viewer's
|
||||
// language; the raw status/body/log_id never leaves the server logs. Connectors
|
||||
// without codes keep the raw text as a Message fallback. Best practice per
|
||||
// Airbyte/Fivetran/Onyx: humanised, actionable, localised UI; raw detail in logs.
|
||||
func fetchFailureSyncError(item *types.FetchedItem, rawMsg string) types.SyncItemError {
|
||||
e := types.SyncItemError{Title: item.Title}
|
||||
if code := item.Metadata["error_reason_code"]; code != "" {
|
||||
e.Code = code
|
||||
if v := item.Metadata["error_reason_code_value"]; v != "" {
|
||||
e.Params = map[string]string{"code": v}
|
||||
}
|
||||
e.Message = item.Metadata["error_reason"] // fallback if the client lacks the key
|
||||
} else {
|
||||
e.Message = rawMsg
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// applyFetchedItem writes a single fetched item into the knowledge base and
|
||||
// updates result counters. It is the shared core of the batch loop and the
|
||||
// streaming handler so item classification (deleted / empty / ingest outcome)
|
||||
// stays identical across both fetch paths.
|
||||
func (s *DataSourceService) applyFetchedItem(
|
||||
ctx context.Context, ds *types.DataSource, item *types.FetchedItem,
|
||||
tagIDs []string, result *types.SyncResult,
|
||||
) {
|
||||
if item.IsDeleted {
|
||||
if ds.SyncDeletions {
|
||||
// Count only — actual KB deletion is intentionally not performed.
|
||||
// Users manage knowledge removal explicitly via the KB UI to avoid
|
||||
// accidental data loss from connector misdetection or reconfiguration.
|
||||
result.Deleted++
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(item.Content) == 0 && item.URL == "" {
|
||||
// Check if this is an error item from the connector (failed to fetch content)
|
||||
if errMsg, hasErr := item.Metadata["error"]; hasErr {
|
||||
logger.Warnf(ctx, "item %q (external_id=%s) fetch failed: %s", item.Title, item.ExternalID, errMsg)
|
||||
result.Failed++
|
||||
recordSyncError(result, fetchFailureSyncError(item, errMsg))
|
||||
} else {
|
||||
logger.Infof(ctx, "skipping item %q (external_id=%s): no content or URL", item.Title, item.ExternalID)
|
||||
result.Skipped++
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
isUpdate, err := s.ingestItem(ctx, ds, item, tagIDs)
|
||||
if err != nil {
|
||||
// Duplicate file/URL is not a failure — count as skipped
|
||||
var dupErr *types.DuplicateKnowledgeError
|
||||
if errors.As(err, &dupErr) {
|
||||
logger.Infof(ctx, "item %q (external_id=%s) already exists, skipping", item.Title, item.ExternalID)
|
||||
result.Skipped++
|
||||
} else {
|
||||
logger.Warnf(ctx, "failed to ingest item %q (external_id=%s): %v", item.Title, item.ExternalID, err)
|
||||
result.Failed++
|
||||
recordSyncError(result, types.SyncItemError{
|
||||
Title: item.Title,
|
||||
Code: "ingest_failed",
|
||||
Message: "Ingest failed; see server logs",
|
||||
})
|
||||
}
|
||||
} else if isUpdate {
|
||||
result.Updated++
|
||||
} else {
|
||||
result.Created++
|
||||
}
|
||||
}
|
||||
|
||||
// streamStartCursor decides which cursor a streaming fetch should resume from.
|
||||
// A user-triggered full sync on its first attempt drops the cursor so every
|
||||
// item is re-fetched; a retried full sync (attempt > 0) and every incremental
|
||||
// sync resume from the last persisted checkpoint so a timed-out run converges
|
||||
// instead of restarting from scratch.
|
||||
func streamStartCursor(ds *types.DataSource, forceFull bool, attempt int) (*types.SyncCursor, error) {
|
||||
if forceFull && attempt == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return ds.ParseSyncCursor()
|
||||
}
|
||||
|
||||
// streamSyncHandler adapts a streaming fetch to the knowledge-base ingest path.
|
||||
// Emit ingests each item as it arrives (bounding memory) and Checkpoint persists
|
||||
// the connector cursor plus live progress counts at page boundaries.
|
||||
type streamSyncHandler struct {
|
||||
svc *DataSourceService
|
||||
ds *types.DataSource
|
||||
tagIDs []string
|
||||
result *types.SyncResult
|
||||
syncLog *types.SyncLog
|
||||
}
|
||||
|
||||
// Emit ingests one streamed item. A canceled context aborts the stream so the
|
||||
// connector stops fetching; per-item ingest failures are recorded in result and
|
||||
// do NOT abort (matching the batch loop, which never fails the whole sync for
|
||||
// one bad document).
|
||||
func (h *streamSyncHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
h.result.Total++
|
||||
h.svc.applyFetchedItem(ctx, h.ds, &item, h.tagIDs, h.result)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Checkpoint persists the connector cursor onto the data source and mirrors the
|
||||
// running counts into the sync log so progress survives a crash and the UI can
|
||||
// reflect a long sync mid-flight instead of jumping from 0 to done.
|
||||
func (h *streamSyncHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
||||
if cursor == nil {
|
||||
return nil
|
||||
}
|
||||
cursorJSON, err := cursor.ToJSON()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.ds.LastSyncCursor = cursorJSON
|
||||
if err := h.svc.dsRepo.UpdateSyncState(ctx, h.ds); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Best-effort live progress; a failure here must not abort the sync.
|
||||
h.syncLog.ItemsTotal = h.result.Total
|
||||
h.syncLog.ItemsCreated = h.result.Created
|
||||
h.syncLog.ItemsUpdated = h.result.Updated
|
||||
h.syncLog.ItemsDeleted = h.result.Deleted
|
||||
h.syncLog.ItemsSkipped = h.result.Skipped
|
||||
h.syncLog.ItemsFailed = h.result.Failed
|
||||
if err := h.svc.syncLogRepo.UpdateResult(ctx, h.syncLog); err != nil {
|
||||
logger.Warnf(ctx, "failed to persist sync log progress at checkpoint: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processSyncStreaming runs a sync through a StreamingConnector, ingesting each
|
||||
// item as it arrives and checkpointing progress so the run is memory-bounded and
|
||||
// resumable after a timeout.
|
||||
func (s *DataSourceService) processSyncStreaming(
|
||||
ctx context.Context, sc datasource.StreamingConnector,
|
||||
ds *types.DataSource, syncLog *types.SyncLog,
|
||||
config *types.DataSourceConfig, payload types.DataSourceSyncPayload, wasPaused bool,
|
||||
) error {
|
||||
// Tenant + auto-tag setup must precede fetching because the stream ingests
|
||||
// each item on the fly.
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, ds.TenantID)
|
||||
tenant, err := s.tenantRepo.GetTenantByID(ctx, ds.TenantID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get tenant info: %v", err)
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
|
||||
types.SyncLogStatusFailed, fmt.Sprintf("Failed to get tenant info: %v", err), wasPaused)
|
||||
return err
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
||||
|
||||
autoTagIDs := s.resolveAutoTagIDs(ctx, ds)
|
||||
|
||||
forceFull := payload.ForceFull || ds.SyncMode == types.SyncModeFull
|
||||
attempt, _ := asynq.GetRetryCount(ctx)
|
||||
startCursor, err := streamStartCursor(ds, forceFull, attempt)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to parse sync cursor: %v", err)
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, &types.SyncResult{}, nil,
|
||||
types.SyncLogStatusFailed, fmt.Sprintf("Invalid cursor: %v", err), wasPaused)
|
||||
return err
|
||||
}
|
||||
|
||||
result := &types.SyncResult{}
|
||||
handler := &streamSyncHandler{svc: s, ds: ds, tagIDs: autoTagIDs, result: result, syncLog: syncLog}
|
||||
|
||||
nextCursor, fetchErr := sc.FetchStream(ctx, config, startCursor, handler)
|
||||
if fetchErr != nil {
|
||||
// Progress so far is already checkpointed onto ds.LastSyncCursor; leave
|
||||
// it in place so the Asynq retry resumes from there. Persist counts.
|
||||
logger.Errorf(ctx, "streaming fetch failed: %v", fetchErr)
|
||||
resultJSON, _ := result.ToJSON()
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON,
|
||||
types.SyncLogStatusFailed, fmt.Sprintf("Fetch failed: %v", fetchErr), wasPaused)
|
||||
return fetchErr
|
||||
}
|
||||
|
||||
resultJSON, _ := result.ToJSON()
|
||||
if err := allFetchedItemsFailedError(result); err != nil {
|
||||
logger.Errorf(ctx, "streaming sync failed while processing fetched items: %v", err)
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, types.SyncLogStatusFailed, err.Error(), wasPaused)
|
||||
return err
|
||||
}
|
||||
|
||||
// Persist the final cursor for the next incremental sync.
|
||||
if nextCursor != nil {
|
||||
if cursorJSON, cerr := nextCursor.ToJSON(); cerr == nil {
|
||||
ds.LastSyncCursor = cursorJSON
|
||||
}
|
||||
}
|
||||
ds.LastSyncAt = timePtr(time.Now().UTC())
|
||||
|
||||
// Surface per-document failures as a partial sync (not silent success), so
|
||||
// the sync-log drawer's failure detail explains which docs didn't make it —
|
||||
// the visibility gap behind "status normal but not everything syncs"
|
||||
// (Tencent/WeKnora#2136). Failed nodes were not advanced in the cursor, so
|
||||
// the next run retries them.
|
||||
status := types.SyncLogStatusSuccess
|
||||
errMsg := ""
|
||||
if result.Failed > 0 {
|
||||
status = types.SyncLogStatusPartial
|
||||
errMsg = fmt.Sprintf("%d document(s) failed to sync", result.Failed)
|
||||
}
|
||||
s.updateSyncRunResult(ctx, ds, syncLog, result, resultJSON, status, errMsg, wasPaused)
|
||||
logger.Infof(ctx, "streaming sync completed: ds=%s created=%d updated=%d deleted=%d skipped=%d failed=%d",
|
||||
payload.DataSourceID, result.Created, result.Updated, result.Deleted, result.Skipped, result.Failed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DataSourceService) updateSyncRunResult(
|
||||
ctx context.Context,
|
||||
ds *types.DataSource,
|
||||
@@ -832,7 +1041,7 @@ func allFetchedItemsFailedError(result *types.SyncResult) error {
|
||||
|
||||
detail := ""
|
||||
if len(result.Errors) > 0 {
|
||||
detail = result.Errors[0]
|
||||
detail = result.Errors[0].Display()
|
||||
const maxDetailLen = 500
|
||||
if len(detail) > maxDetailLen {
|
||||
detail = detail[:maxDetailLen] + "..."
|
||||
|
||||
@@ -153,7 +153,7 @@ func TestAllFetchedItemsFailedError(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 2,
|
||||
Failed: 2,
|
||||
Errors: []string{"doc one: export failed"},
|
||||
Errors: []types.SyncItemError{{Message: "doc one: export failed"}},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "all fetched items failed during sync (2/2)")
|
||||
@@ -181,7 +181,7 @@ func TestAllFetchedItemsFailedErrorTruncatesLongDetail(t *testing.T) {
|
||||
err := allFetchedItemsFailedError(&types.SyncResult{
|
||||
Total: 1,
|
||||
Failed: 1,
|
||||
Errors: []string{strings.Repeat("x", 600)},
|
||||
Errors: []types.SyncItemError{{Message: strings.Repeat("x", 600)}},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.LessOrEqual(t, len(err.Error()), 560)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// recordingDSRepo captures UpdateSyncState calls so checkpoint persistence can
|
||||
// be asserted without a database.
|
||||
type recordingDSRepo struct {
|
||||
kbDeleteDSRepo
|
||||
updated []*types.DataSource
|
||||
}
|
||||
|
||||
func (r *recordingDSRepo) UpdateSyncState(_ context.Context, ds *types.DataSource) error {
|
||||
// Snapshot the fields a checkpoint is expected to persist.
|
||||
cp := *ds
|
||||
r.updated = append(r.updated, &cp)
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeConnectorCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) types.JSON {
|
||||
t.Helper()
|
||||
inner := map[string]interface{}{"space_node_times": spaceNodeTimes}
|
||||
b, err := json.Marshal(&types.SyncCursor{ConnectorCursor: inner})
|
||||
require.NoError(t, err)
|
||||
return types.JSON(b)
|
||||
}
|
||||
|
||||
// A fresh full sync (ForceFull, first attempt) must ignore any recorded cursor
|
||||
// and re-fetch everything. A retry of that same task (attempt > 0) must instead
|
||||
// resume from the checkpointed cursor so it converges instead of restarting.
|
||||
func TestStreamStartCursor_ForceFullFirstAttemptDropsCursor(t *testing.T) {
|
||||
ds := &types.DataSource{
|
||||
LastSyncCursor: makeConnectorCursor(t, map[string]map[string]string{"space1": {"nt1": "100"}}),
|
||||
}
|
||||
|
||||
fresh, err := streamStartCursor(ds, true /*forceFull*/, 0 /*attempt*/)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, fresh, "fresh ForceFull must drop the cursor to re-fetch everything")
|
||||
|
||||
retry, err := streamStartCursor(ds, true /*forceFull*/, 1 /*attempt*/)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, retry, "a retried ForceFull must resume from the checkpoint")
|
||||
assert.NotNil(t, retry.ConnectorCursor["space_node_times"])
|
||||
}
|
||||
|
||||
// Incremental sync always resumes from the recorded cursor regardless of attempt.
|
||||
func TestStreamStartCursor_IncrementalKeepsCursor(t *testing.T) {
|
||||
ds := &types.DataSource{
|
||||
LastSyncCursor: makeConnectorCursor(t, map[string]map[string]string{"space1": {"nt1": "100"}}),
|
||||
}
|
||||
cur, err := streamStartCursor(ds, false /*forceFull*/, 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cur)
|
||||
assert.NotNil(t, cur.ConnectorCursor["space_node_times"])
|
||||
}
|
||||
|
||||
func newStreamHandler(svc *DataSourceService, ds *types.DataSource, result *types.SyncResult, syncLog *types.SyncLog) *streamSyncHandler {
|
||||
return &streamSyncHandler{svc: svc, ds: ds, result: result, syncLog: syncLog}
|
||||
}
|
||||
|
||||
// Emit routes items through the same classification as the batch loop: deleted
|
||||
// items are counted, and connector-reported failures (an item carrying only a
|
||||
// Metadata["error"]) land in result.Failed with a message — never silently lost.
|
||||
func TestStreamHandler_EmitClassifiesDeletedAndFailed(t *testing.T) {
|
||||
ds := &types.DataSource{ID: "ds-1", Type: types.ConnectorTypeFeishu, SyncDeletions: true}
|
||||
result := &types.SyncResult{}
|
||||
h := newStreamHandler(&DataSourceService{}, ds, result, &types.SyncLog{})
|
||||
|
||||
require.NoError(t, h.Emit(context.Background(), types.FetchedItem{ExternalID: "gone", IsDeleted: true}))
|
||||
require.NoError(t, h.Emit(context.Background(), types.FetchedItem{
|
||||
ExternalID: "bad", Title: "Broken Doc",
|
||||
Metadata: map[string]string{"error": "export failed"},
|
||||
}))
|
||||
|
||||
assert.Equal(t, 1, result.Deleted)
|
||||
assert.Equal(t, 1, result.Failed)
|
||||
require.Len(t, result.Errors, 1)
|
||||
assert.Equal(t, "Broken Doc", result.Errors[0].Title)
|
||||
assert.Contains(t, result.Errors[0].Message, "export failed")
|
||||
}
|
||||
|
||||
// A canceled context aborts the stream: Emit returns the context error so the
|
||||
// connector stops fetching instead of burning API budget on a doomed run.
|
||||
func TestStreamHandler_EmitAbortsOnCanceledContext(t *testing.T) {
|
||||
h := newStreamHandler(&DataSourceService{}, &types.DataSource{}, &types.SyncResult{}, &types.SyncLog{})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := h.Emit(ctx, types.FetchedItem{ExternalID: "x", Content: []byte("data"), FileName: "x.md"})
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
// Checkpoint persists the connector cursor onto the data source so a crash
|
||||
// after it keeps the progress made so far.
|
||||
func TestStreamHandler_CheckpointPersistsCursor(t *testing.T) {
|
||||
dsRepo := &recordingDSRepo{}
|
||||
svc := &DataSourceService{dsRepo: dsRepo, syncLogRepo: &processSyncSyncLogRepo{logs: map[string]*types.SyncLog{}}}
|
||||
ds := &types.DataSource{ID: "ds-1"}
|
||||
result := &types.SyncResult{Created: 3}
|
||||
syncLog := &types.SyncLog{ID: "log-1"}
|
||||
h := newStreamHandler(svc, ds, result, syncLog)
|
||||
|
||||
cursor := &types.SyncCursor{ConnectorCursor: map[string]interface{}{
|
||||
"space_node_times": map[string]map[string]string{"space1": {"nt1": "100"}},
|
||||
}}
|
||||
require.NoError(t, h.Checkpoint(context.Background(), cursor))
|
||||
|
||||
require.Len(t, dsRepo.updated, 1)
|
||||
assert.NotEmpty(t, dsRepo.updated[0].LastSyncCursor, "checkpoint must persist the cursor JSON")
|
||||
}
|
||||
@@ -51,6 +51,48 @@ type Connector interface {
|
||||
FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error)
|
||||
}
|
||||
|
||||
// StreamHandler receives items and progress checkpoints emitted during a
|
||||
// streaming fetch. The service implements it to ingest each item as it arrives
|
||||
// (bounding memory to one item instead of the whole wiki) and to persist the
|
||||
// connector cursor at page boundaries, so a sync that times out mid-traversal
|
||||
// resumes from the last checkpoint instead of restarting from scratch
|
||||
// (Tencent/WeKnora#2136).
|
||||
type StreamHandler interface {
|
||||
// Emit ingests a single fetched item. Returning an error aborts the
|
||||
// stream: the connector stops fetching and propagates the error, since a
|
||||
// failed ingest means the sync is failing and further API calls are wasted.
|
||||
Emit(ctx context.Context, item types.FetchedItem) error
|
||||
|
||||
// Checkpoint persists the cursor reached so far. The cursor is only valid
|
||||
// for the duration of the call (the connector may keep mutating its backing
|
||||
// maps afterwards), so implementations must serialize it synchronously.
|
||||
//
|
||||
// The cursor MUST be a complete resumable snapshot, not a delta: resuming
|
||||
// from it must reproduce all progress so far. This is what lets the service
|
||||
// treat a checkpoint as a safe restart point and, for a full sync, drop the
|
||||
// prior baseline without losing already-synced state.
|
||||
Checkpoint(ctx context.Context, cursor *types.SyncCursor) error
|
||||
}
|
||||
|
||||
// StreamingConnector is an optional interface. Connectors that implement it let
|
||||
// the service interleave fetch→ingest→checkpoint so a large sync persists
|
||||
// incrementally and resumes after a timeout, rather than holding every item in
|
||||
// memory and losing all progress on retry. Connectors that do not implement it
|
||||
// fall back to FetchAll / FetchIncremental unchanged.
|
||||
type StreamingConnector interface {
|
||||
Connector
|
||||
|
||||
// FetchStream walks the configured resources starting from cursor (nil =
|
||||
// from the beginning / full sync), calling h.Emit for each changed item and
|
||||
// h.Checkpoint at page boundaries. It returns the final cursor for the next
|
||||
// sync. Nodes already recorded in cursor at their current edit time are
|
||||
// skipped, which is what makes a resumed sync converge.
|
||||
FetchStream(
|
||||
ctx context.Context, config *types.DataSourceConfig,
|
||||
cursor *types.SyncCursor, h StreamHandler,
|
||||
) (*types.SyncCursor, error)
|
||||
}
|
||||
|
||||
// ConnectorRegistry manages the registration and lookup of available connectors
|
||||
type ConnectorRegistry struct {
|
||||
connectors map[string]Connector
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -117,58 +118,161 @@ func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
// doRequest executes an authenticated API request and decodes the JSON response.
|
||||
// Retry policy shared by doRequest (JSON API calls) and downloadRawBytes (file
|
||||
// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off.
|
||||
const (
|
||||
feishuMaxRetries = 3
|
||||
feishuMax5xxRetries = 1
|
||||
feishuRetry5xxDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
|
||||
|
||||
// doRequest executes an authenticated API request and decodes the JSON response,
|
||||
// retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive
|
||||
// export/wiki APIs are aggressively rate limited, and a thousand-document sync
|
||||
// issues tens of thousands of calls; without backoff a single 429 burst used to
|
||||
// fail whole swathes of documents silently. 429 responses honour Retry-After;
|
||||
// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx).
|
||||
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
|
||||
const (
|
||||
maxRetries = feishuMaxRetries
|
||||
max5xxRetries = feishuMax5xxRetries
|
||||
retry5xxDelay = feishuRetry5xxDelay
|
||||
)
|
||||
backoff := feishuRetryBackoff
|
||||
|
||||
token, err := c.getTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
var bodyBytes []byte
|
||||
if body != nil {
|
||||
bodyBytes, err := json.Marshal(body)
|
||||
bodyBytes, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
var lastErr error
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read body once for logging + decoding
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
var bodyReader io.Reader
|
||||
if bodyBytes != nil {
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] %s %s (retry %d/%d)", method, path, attempt, maxRetries)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("execute request: %w", err)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
respBody, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read response body: %w", readErr)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)])
|
||||
lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500))
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
lastErr = fmt.Errorf("feishu server error: status=%d body=%s", resp.StatusCode, truncate(string(respBody), 500))
|
||||
if attempt < max5xxRetries {
|
||||
if sErr := sleepCtx(ctx, retry5xxDelay); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// parseRetryAfter interprets a Retry-After header value (seconds) into a wait
|
||||
// duration, coercing 0/negative to a short delay and falling back when absent
|
||||
// or unparseable.
|
||||
func parseRetryAfter(header string, fallback time.Duration) time.Duration {
|
||||
if header == "" {
|
||||
return fallback
|
||||
}
|
||||
secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if secs <= 0 {
|
||||
return 100 * time.Millisecond
|
||||
}
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
|
||||
// sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the
|
||||
// context ends first so retries abort promptly on task cancellation/timeout.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen and appends "..." if truncated.
|
||||
@@ -554,31 +658,83 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
var lastErr error
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
for attempt := 0; attempt <= feishuMaxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s (retry %d/%d)", path, attempt, feishuMaxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("download request: %w", err)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)])
|
||||
lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500))
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("download server error: status=%d body=%s", resp.StatusCode, truncate(string(body), 500))
|
||||
if attempt < feishuMax5xxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetry5xxDelay); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read download body: %w", readErr)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// retryTestServer builds a server that always answers the auth-token call and
|
||||
// routes the given target path to h, so tests can drive doRequest's retry loop.
|
||||
func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
TenantAccessToken: "fake-token",
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
mux.HandleFunc(target, h)
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}
|
||||
}
|
||||
|
||||
func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
// "0" is coerced to a short delay inside the client so the test stays fast.
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
return
|
||||
}
|
||||
writeJSON(w, apiResponse{Code: 0})
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
var resp apiResponse
|
||||
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_429ExhaustsRetries(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when 429s exceed the retry budget")
|
||||
}
|
||||
if attempts != 4 { // initial + 3 retries
|
||||
t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_5xxRetriesOnce(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.doRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error after 5xx exhaustion")
|
||||
}
|
||||
if attempts != 2 { // initial + 1 retry
|
||||
t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error on 400")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("payload-bytes"))
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
data, err := c.downloadRawBytes(context.Background(), "/dl")
|
||||
if err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if string(data) != "payload-bytes" {
|
||||
t.Errorf("data = %q, want %q", string(data), "payload-bytes")
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil {
|
||||
t.Fatal("expected error on 403")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
fallback := 5 * time.Second
|
||||
tests := []struct {
|
||||
header string
|
||||
want time.Duration
|
||||
}{
|
||||
{"", fallback},
|
||||
{"0", 100 * time.Millisecond},
|
||||
{"-1", 100 * time.Millisecond}, // negative coerced to a short delay
|
||||
{"3", 3 * time.Second},
|
||||
{"abc", fallback}, // unparseable
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := parseRetryAfter(tt.header, fallback); got != tt.want {
|
||||
t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -27,6 +28,10 @@ func NewConnector(region Region) *Connector {
|
||||
return &Connector{region: region}
|
||||
}
|
||||
|
||||
// Feishu supports resumable streaming sync; the service prefers FetchStream over
|
||||
// FetchAll/FetchIncremental when a connector implements StreamingConnector.
|
||||
var _ datasource.StreamingConnector = (*Connector)(nil)
|
||||
|
||||
// Type returns the connector type identifier.
|
||||
func (c *Connector) Type() string {
|
||||
return c.region.ConnectorType
|
||||
@@ -34,6 +39,51 @@ func (c *Connector) Type() string {
|
||||
|
||||
const feishuWikiNodeResourceSeparator = ":"
|
||||
|
||||
// feishuStreamCheckpointInterval is how many processed nodes pass between
|
||||
// cursor checkpoints during a streaming fetch. Small enough that a timed-out
|
||||
// sync loses little work on resume, large enough that checkpoint persistence
|
||||
// (a DB write) does not dominate. Overridable in tests. See FetchStream.
|
||||
var feishuStreamCheckpointInterval = 50
|
||||
|
||||
// feishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as
|
||||
// well as node count. Without it, a sync of fewer than
|
||||
// feishuStreamCheckpointInterval very slow (rate-limited) exports could reach
|
||||
// the 2h task timeout having never checkpointed, and resume from scratch every
|
||||
// retry — the #2136 "never fully syncs" case. Overridable in tests.
|
||||
var feishuStreamCheckpointMaxInterval = 30 * time.Second
|
||||
|
||||
// fetchTally accumulates the outcome of fetching a wiki node subtree so the
|
||||
// connector can emit a single actionable summary. Without it, unsupported nodes
|
||||
// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users
|
||||
// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136).
|
||||
type fetchTally struct {
|
||||
discovered int
|
||||
fetched int
|
||||
failed int
|
||||
skippedByType map[string]int
|
||||
}
|
||||
|
||||
func newFetchTally(discovered int) *fetchTally {
|
||||
return &fetchTally{discovered: discovered, skippedByType: map[string]int{}}
|
||||
}
|
||||
|
||||
func (t *fetchTally) fetch() { t.fetched++ }
|
||||
func (t *fetchTally) fail() { t.failed++ }
|
||||
func (t *fetchTally) skip(objType string) { t.skippedByType[objType]++ }
|
||||
|
||||
func (t *fetchTally) skipped() int {
|
||||
n := 0
|
||||
for _, c := range t.skippedByType {
|
||||
n += c
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (t *fetchTally) summary() string {
|
||||
return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v",
|
||||
t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType)
|
||||
}
|
||||
|
||||
// Validate verifies that the Feishu configuration is valid by testing connectivity.
|
||||
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
feishuConfig, err := parseFeishuConfig(config, c.region)
|
||||
@@ -184,25 +234,35 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
|
||||
allItems = appendWikiNodeListFailureItems(allItems, spaceID, resourceID, partialErr.Failures)
|
||||
}
|
||||
|
||||
// Fetch content for each document node
|
||||
for _, node := range nodes {
|
||||
// Fetch content for each document node, tallying outcomes so a single
|
||||
// summary line explains where every discovered node went.
|
||||
tally := newFetchTally(len(nodes))
|
||||
for i, node := range nodes {
|
||||
item, err := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
|
||||
if err != nil {
|
||||
tally.fail()
|
||||
// Log error but continue with other nodes
|
||||
allItems = append(allItems, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
Metadata: feishuErrorItemMeta(err, nil),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if item != nil {
|
||||
tally.fetch()
|
||||
allItems = append(allItems, *item)
|
||||
} else {
|
||||
// Unsupported obj_type (mindnote/slides/…): skipped with no item.
|
||||
tally.skip(node.ObjType)
|
||||
}
|
||||
if n := i + 1; n%100 == 0 {
|
||||
logger.Infof(ctx, "[Feishu] sync progress resource=%s %d/%d (%s)",
|
||||
resourceID, n, len(nodes), tally.summary())
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[Feishu] sync summary resource=%s %s", resourceID, tally.summary())
|
||||
}
|
||||
|
||||
return allItems, nil
|
||||
@@ -293,9 +353,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
Metadata: feishuErrorItemMeta(err, nil),
|
||||
})
|
||||
continue
|
||||
}
|
||||
@@ -334,6 +392,246 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
return changedItems, nextSyncCursor, nil
|
||||
}
|
||||
|
||||
// FetchStream performs a resumable, memory-bounded sync. It unifies the full
|
||||
// and incremental paths: with cursor == nil it fetches everything, and with a
|
||||
// cursor it skips nodes whose recorded edit time is unchanged — the same
|
||||
// mechanism that lets a sync which timed out mid-traversal resume from the last
|
||||
// checkpoint instead of restarting (Tencent/WeKnora#2136).
|
||||
//
|
||||
// Instead of accumulating every item in memory (FetchAll), it Emits each item
|
||||
// as it is fetched and Checkpoints the cursor every feishuStreamCheckpointInterval
|
||||
// processed nodes, so progress is durable across the Asynq task's 2h timeout.
|
||||
func (c *Connector) FetchStream(
|
||||
ctx context.Context, config *types.DataSourceConfig,
|
||||
cursor *types.SyncCursor, h datasource.StreamHandler,
|
||||
) (*types.SyncCursor, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config, c.region)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := NewClient(feishuConfig)
|
||||
|
||||
var prevCursor feishuCursor
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(cursorBytes, &prevCursor)
|
||||
}
|
||||
|
||||
newCursor := feishuCursor{
|
||||
LastSyncTime: time.Now(),
|
||||
SpaceNodeTimes: make(map[string]map[string]string),
|
||||
}
|
||||
|
||||
resourceIDs := config.ResourceIDs
|
||||
if len(resourceIDs) == 0 {
|
||||
return nil, fmt.Errorf("no resource IDs (wiki space IDs or wiki node IDs) configured")
|
||||
}
|
||||
|
||||
processed := 0
|
||||
lastCheckpoint := time.Now()
|
||||
for _, resourceID := range resourceIDs {
|
||||
spaceID, nodeToken := parseWikiResourceID(resourceID)
|
||||
nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken)
|
||||
var partialErr *partialWikiNodeListError
|
||||
if err != nil {
|
||||
if !errors.As(err, &partialErr) {
|
||||
return nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err)
|
||||
}
|
||||
for _, item := range appendWikiNodeListFailureItems(nil, spaceID, resourceID, partialErr.Failures) {
|
||||
if eerr := h.Emit(ctx, item); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newCursor.SpaceNodeTimes[resourceID] = make(map[string]string)
|
||||
// On a partial listing, carry prior edit times forward so a later full
|
||||
// listing can still detect changes and deletions.
|
||||
if partialErr != nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
for tok, et := range prevTimes {
|
||||
newCursor.SpaceNodeTimes[resourceID][tok] = et
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentNodes := make(map[string]bool)
|
||||
tally := newFetchTally(len(nodes))
|
||||
for i, node := range nodes {
|
||||
currentNodes[node.NodeToken] = true
|
||||
editTimeStr := node.ObjEditTime
|
||||
if editTimeStr == "" {
|
||||
editTimeStr = node.NodeEditTime
|
||||
}
|
||||
|
||||
// Prior recorded edit time for this node, if any.
|
||||
var prevEdit string
|
||||
var hadPrev bool
|
||||
if prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
prevEdit, hadPrev = prevTimes[node.NodeToken]
|
||||
}
|
||||
}
|
||||
|
||||
// Resume/incremental fast-path: a node recorded at its current edit
|
||||
// time is unchanged (or already synced this run) — keep the record
|
||||
// and skip re-fetching.
|
||||
if hadPrev && prevEdit == editTimeStr {
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
|
||||
continue
|
||||
}
|
||||
|
||||
item, ferr := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
|
||||
if ferr != nil {
|
||||
tally.fail()
|
||||
// Do NOT advance the cursor: the content was never fetched.
|
||||
// Retain the prior edit time (if any) so prev != current next
|
||||
// run and the node is retried, instead of being permanently
|
||||
// skipped on a transient export failure (Tencent/WeKnora#2136).
|
||||
if hadPrev {
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = prevEdit
|
||||
}
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: feishuErrorItemMeta(ferr, nil),
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
} else {
|
||||
// Fetched, or an unsupported obj_type (nothing to fetch): record
|
||||
// the current edit time so the node is not re-processed next run.
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
|
||||
if item != nil {
|
||||
tally.fetch()
|
||||
if eerr := h.Emit(ctx, *item); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
} else {
|
||||
// Unsupported obj_type (mindnote/slides/…): no item.
|
||||
tally.skip(node.ObjType)
|
||||
}
|
||||
}
|
||||
|
||||
processed++
|
||||
if processed%feishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= feishuStreamCheckpointMaxInterval {
|
||||
if cerr := h.Checkpoint(ctx, newCursor.toSyncCursor()); cerr != nil {
|
||||
logger.Warnf(ctx, "[Feishu] stream checkpoint failed: %v", cerr)
|
||||
}
|
||||
lastCheckpoint = time.Now()
|
||||
}
|
||||
if n := i + 1; n%100 == 0 {
|
||||
logger.Infof(ctx, "[Feishu] stream progress resource=%s %d/%d (%s)",
|
||||
resourceID, n, len(nodes), tally.summary())
|
||||
}
|
||||
}
|
||||
|
||||
// Detect deleted nodes (only when the full tree was listed successfully).
|
||||
if partialErr == nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
for tok := range prevTimes {
|
||||
if !currentNodes[tok] {
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: tok,
|
||||
IsDeleted: true,
|
||||
SourceResourceID: resourceID,
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[Feishu] stream summary resource=%s %s", resourceID, tally.summary())
|
||||
}
|
||||
|
||||
return newCursor.toSyncCursor(), nil
|
||||
}
|
||||
|
||||
// toSyncCursor converts the connector-specific feishuCursor into the generic
|
||||
// SyncCursor persisted by the service. It marshals through JSON so the returned
|
||||
// value is a snapshot, decoupled from later mutation of the connector's maps.
|
||||
func (fc feishuCursor) toSyncCursor() *types.SyncCursor {
|
||||
m := make(map[string]interface{})
|
||||
cursorBytes, _ := json.Marshal(fc)
|
||||
_ = json.Unmarshal(cursorBytes, &m)
|
||||
return &types.SyncCursor{
|
||||
LastSyncTime: fc.LastSyncTime,
|
||||
ConnectorCursor: m,
|
||||
}
|
||||
}
|
||||
|
||||
var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`)
|
||||
|
||||
// feishuErrorCode extracts the numeric Feishu error code from a raw error string
|
||||
// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort.
|
||||
func feishuErrorCode(raw string) string {
|
||||
if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (mapped to a localized string on the frontend), an optional numeric Feishu
|
||||
// error code for interpolation, and an English fallback message for clients
|
||||
// without the i18n key. The raw status/JSON body/log_id is never returned here —
|
||||
// it stays in the server logs. Dumping it in the UI is the anti-pattern
|
||||
// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync
|
||||
// (the cursor is retained); auth/permission errors point at the fix instead.
|
||||
func feishuFailure(err error) (code, codeValue, fallback string) {
|
||||
if err == nil {
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
|
||||
switch {
|
||||
case strings.Contains(s, "auth error"),
|
||||
strings.Contains(s, "invalid access token"),
|
||||
strings.Contains(s, "permission"),
|
||||
strings.Contains(s, "forbidden"),
|
||||
strings.Contains(s, "status=403"):
|
||||
return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes"
|
||||
case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"):
|
||||
return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync"
|
||||
case strings.Contains(s, "timed out"),
|
||||
strings.Contains(s, "timeout"),
|
||||
strings.Contains(s, "deadline exceeded"):
|
||||
return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"
|
||||
case strings.Contains(s, "server error"):
|
||||
return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync"
|
||||
case strings.Contains(s, "api error"),
|
||||
strings.Contains(s, "export task failed"),
|
||||
strings.Contains(s, "download failed"):
|
||||
if v := feishuErrorCode(err.Error()); v != "" {
|
||||
return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v)
|
||||
}
|
||||
return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync"
|
||||
default:
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
}
|
||||
|
||||
// feishuErrorItemMeta builds the metadata for a failed item: the raw error (for
|
||||
// server logs) plus the classified i18n code / param / fallback (for a
|
||||
// localisable SyncItemError in the UI), merged with any caller-supplied extras.
|
||||
func feishuErrorItemMeta(err error, extra map[string]string) map[string]string {
|
||||
code, codeValue, fallback := feishuFailure(err)
|
||||
m := map[string]string{
|
||||
"error": err.Error(),
|
||||
"error_reason_code": code,
|
||||
"error_reason": fallback,
|
||||
}
|
||||
if codeValue != "" {
|
||||
m["error_reason_code_value"] = codeValue
|
||||
}
|
||||
for k, v := range extra {
|
||||
m[k] = v
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, resourceID string, failures []wikiNodeListFailure) []types.FetchedItem {
|
||||
for _, failure := range failures {
|
||||
node := failure.Node
|
||||
@@ -345,13 +643,12 @@ func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, r
|
||||
ExternalID: node.NodeToken,
|
||||
Title: title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": failure.Err.Error(),
|
||||
Metadata: feishuErrorItemMeta(failure.Err, map[string]string{
|
||||
"channel": types.ChannelFeishu,
|
||||
"node_token": node.NodeToken,
|
||||
"space_id": spaceID,
|
||||
"failure_stage": "list_children",
|
||||
},
|
||||
}),
|
||||
})
|
||||
}
|
||||
return items
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// statefulFeishu is a fake Feishu server whose per-node export outcome can be
|
||||
// changed between sync passes. A node whose obj_token is in failTokens has its
|
||||
// export status poll return a failed job, so fetchNodeContent errors for it —
|
||||
// modelling a transient rate-limit / export failure that later recovers.
|
||||
type statefulFeishu struct {
|
||||
mu sync.Mutex
|
||||
failTokens map[string]bool
|
||||
}
|
||||
|
||||
func (s *statefulFeishu) setFail(tokens ...string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.failTokens = map[string]bool{}
|
||||
for _, t := range tokens {
|
||||
s.failTokens[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statefulFeishu) shouldFail(token string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.failTokens[token]
|
||||
}
|
||||
|
||||
func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFeishu) {
|
||||
s := &statefulFeishu{failTokens: map[string]bool{}}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiSpaceListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
|
||||
})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: nodes},
|
||||
})
|
||||
})
|
||||
// Export create: ticket == obj_token so the poll below can key on it.
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
writeJSON(w, exportTaskCreateResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct{ Ticket string `json:"ticket"` }{Ticket: body.Token},
|
||||
})
|
||||
})
|
||||
// Export status poll: /open-apis/drive/v1/export_tasks/<ticket>?token=<objToken>
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/file/") && strings.HasSuffix(r.URL.Path, "/download") {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("fake-docx-content"))
|
||||
return
|
||||
}
|
||||
token := r.URL.Query().Get("token")
|
||||
status := 0
|
||||
if s.shouldFail(token) {
|
||||
status = 3 // failed job
|
||||
}
|
||||
resp := exportTaskStatusResponse{apiResponse: apiResponse{Code: 0}}
|
||||
resp.Data.Result.FileToken = "file-" + token
|
||||
resp.Data.Result.FileName = "exported.docx"
|
||||
resp.Data.Result.JobStatus = status
|
||||
resp.Data.Result.JobErrorMsg = "rate limited"
|
||||
writeJSON(w, resp)
|
||||
})
|
||||
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}, s
|
||||
}
|
||||
|
||||
// convergenceHandler models the service side across a resumable sync:
|
||||
// - Emit ingests content items (mirroring the service's ctx check + apply),
|
||||
// records failure items separately, and can simulate a mid-tree task timeout
|
||||
// by canceling the run's context after cancelAfterCalls Emit calls.
|
||||
// - Checkpoint snapshots the connector cursor (the DB round-trip).
|
||||
type convergenceHandler struct {
|
||||
ingested []string // ExternalIDs of successfully ingested content items
|
||||
failed []string // ExternalIDs surfaced as failures (Metadata["error"])
|
||||
exportedTokens map[string]int
|
||||
checkpoints []feishuCursor
|
||||
calls int
|
||||
cancelAfterCall int // >0: cancel ctx and abort on the Nth Emit call
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (h *convergenceHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if err := ctx.Err(); err != nil { // mirror service Emit
|
||||
return err
|
||||
}
|
||||
h.calls++
|
||||
if h.cancelAfterCall > 0 && h.calls >= h.cancelAfterCall {
|
||||
if h.cancel != nil {
|
||||
h.cancel()
|
||||
}
|
||||
return context.Canceled // service returns ctx err → connector aborts
|
||||
}
|
||||
if item.Metadata["error"] != "" {
|
||||
h.failed = append(h.failed, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
h.ingested = append(h.ingested, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *convergenceHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
h.checkpoints = append(h.checkpoints, fc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func lastCheckpointCursor(h *convergenceHandler, t *testing.T) *types.SyncCursor {
|
||||
t.Helper()
|
||||
if len(h.checkpoints) == 0 {
|
||||
return nil
|
||||
}
|
||||
last := h.checkpoints[len(h.checkpoints)-1]
|
||||
b, _ := json.Marshal(last)
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatalf("rebuild persisted cursor: %v", err)
|
||||
}
|
||||
return &types.SyncCursor{ConnectorCursor: m}
|
||||
}
|
||||
|
||||
// TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure is the
|
||||
// end-to-end proof for Tencent/WeKnora#2136: a large-ish wiki that (a) hits a
|
||||
// transient per-node export failure and (b) is killed by the 2h task timeout
|
||||
// mid-traversal must, on the asynq retry, resume from the last checkpoint,
|
||||
// re-fetch only what is outstanding, retry the transiently-failed node, and end
|
||||
// with EVERY document synced exactly once — no permanent skip, no full restart,
|
||||
// no redundant re-export of already-done nodes.
|
||||
func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T) {
|
||||
// Checkpoint on every processed node so the persisted cursor is precise.
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1
|
||||
defer func() { feishuStreamCheckpointInterval = prevN }()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", ObjEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", ObjEditTime: "200"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "docx", Title: "Doc4", ObjEditTime: "400"},
|
||||
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "docx", Title: "Doc5", ObjEditTime: "500"},
|
||||
}
|
||||
ts, cfg, srv := newStatefulFeishu(nodes)
|
||||
defer ts.Close()
|
||||
cfgDS := makeConfig(cfg, []string{"space1"})
|
||||
c := NewConnector(RegionFeishu)
|
||||
|
||||
// ---- Pass 1: obj2 export fails transiently; task "times out" on the 3rd
|
||||
// Emit call (nt1 success, nt2 failure-item, then cancel as nt3 is emitted).
|
||||
srv.setFail("obj2")
|
||||
ctx1, cancel1 := context.WithCancel(context.Background())
|
||||
h1 := &convergenceHandler{cancelAfterCall: 3, cancel: cancel1}
|
||||
_, err1 := c.FetchStream(ctx1, cfgDS, nil, h1)
|
||||
cancel1()
|
||||
if err1 == nil {
|
||||
t.Fatalf("pass 1 expected a timeout/abort error, got nil")
|
||||
}
|
||||
|
||||
// Pass 1 ingested only nt1; nt2 surfaced as a failure; nt3+ never reached.
|
||||
if got := strings.Join(h1.ingested, ","); got != "nt1" {
|
||||
t.Fatalf("pass 1 ingested = %q, want just nt1", got)
|
||||
}
|
||||
|
||||
// The persisted cursor (last checkpoint) must contain nt1 but NOT the
|
||||
// aborted nt3, nor the transiently-failed nt2 — otherwise resume would skip
|
||||
// them forever.
|
||||
persisted := lastCheckpointCursor(h1, t)
|
||||
if persisted == nil {
|
||||
t.Fatalf("pass 1 wrote no checkpoint — resume would restart from scratch")
|
||||
}
|
||||
var pc feishuCursor
|
||||
pb, _ := json.Marshal(persisted.ConnectorCursor)
|
||||
_ = json.Unmarshal(pb, &pc)
|
||||
p := pc.SpaceNodeTimes["space1"]
|
||||
if _, ok := p["nt1"]; !ok {
|
||||
t.Errorf("persisted cursor missing nt1 (done work lost on resume)")
|
||||
}
|
||||
if _, ok := p["nt2"]; ok {
|
||||
t.Errorf("persisted cursor recorded transiently-failed nt2 — it would be skipped forever")
|
||||
}
|
||||
if _, ok := p["nt3"]; ok {
|
||||
t.Errorf("persisted cursor recorded aborted nt3 — it would be skipped forever")
|
||||
}
|
||||
|
||||
// ---- Pass 2: asynq retry. obj2 has recovered; run to completion resuming
|
||||
// from the persisted cursor.
|
||||
srv.setFail() // clear all failures
|
||||
h2 := &convergenceHandler{}
|
||||
next2, err2 := c.FetchStream(context.Background(), cfgDS, persisted, h2)
|
||||
if err2 != nil {
|
||||
t.Fatalf("pass 2 error: %v", err2)
|
||||
}
|
||||
|
||||
// nt1 was already done → must be skipped (not re-exported, not re-ingested).
|
||||
for _, id := range h2.ingested {
|
||||
if id == "nt1" {
|
||||
t.Errorf("pass 2 re-ingested nt1 — redundant re-export of completed work")
|
||||
}
|
||||
}
|
||||
// Everything still outstanding (nt2 transiently-failed, nt3 aborted, nt4, nt5)
|
||||
// must be ingested in pass 2.
|
||||
want := map[string]bool{"nt2": true, "nt3": true, "nt4": true, "nt5": true}
|
||||
got2 := map[string]bool{}
|
||||
for _, id := range h2.ingested {
|
||||
got2[id] = true
|
||||
}
|
||||
for id := range want {
|
||||
if !got2[id] {
|
||||
t.Errorf("pass 2 did not ingest %s (want all outstanding nodes)", id)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Convergence: union across both passes covers ALL five nodes exactly.
|
||||
union := map[string]bool{}
|
||||
for _, id := range h1.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, id := range h2.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if !union[n.NodeToken] {
|
||||
t.Errorf("node %s was NEVER synced across both passes — data loss (#2136)", n.NodeToken)
|
||||
}
|
||||
}
|
||||
if len(union) != len(nodes) {
|
||||
t.Errorf("synced %d distinct nodes, want %d", len(union), len(nodes))
|
||||
}
|
||||
|
||||
// Final cursor is a complete snapshot of every node (next incremental sync
|
||||
// starts clean).
|
||||
var fc feishuCursor
|
||||
nb, _ := json.Marshal(next2.ConnectorCursor)
|
||||
_ = json.Unmarshal(nb, &fc)
|
||||
for _, n := range nodes {
|
||||
if _, ok := fc.SpaceNodeTimes["space1"][n.NodeToken]; !ok {
|
||||
t.Errorf("final cursor missing %s — incremental sync would re-fetch it", n.NodeToken)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (+ optional numeric feishu code) and an English fallback message. The frontend
|
||||
// localises by code; the raw status/body/log_id stays in the server logs and
|
||||
// must never appear in the code, codeValue or fallback.
|
||||
func TestFeishuFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
wantCode string
|
||||
wantCodeValue string
|
||||
fallbackHas []string
|
||||
noLeak []string // must appear in neither code nor fallback
|
||||
}{
|
||||
{
|
||||
name: "rate limited",
|
||||
err: errors.New("feishu rate limited: status=429 body={\"code\":99991400,\"msg\":\"too many request\"}"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"retry"},
|
||||
noLeak: []string{"body=", "{", "99991400"},
|
||||
},
|
||||
{
|
||||
name: "server 5xx",
|
||||
err: errors.New("feishu server error: status=500 body={\"code\":1663}"),
|
||||
wantCode: "feishu_server_unavailable",
|
||||
noLeak: []string{"body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "export timeout",
|
||||
err: errors.New("export 季度报告 (docx): export task timed out after 60s (ticket=abc)"),
|
||||
wantCode: "feishu_timeout",
|
||||
noLeak: []string{"ticket=", "abc"},
|
||||
},
|
||||
{
|
||||
name: "api error carries the feishu code as a param",
|
||||
err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"error\":{\"log_id\":\"20260\"}}"),
|
||||
wantCode: "feishu_api_error",
|
||||
wantCodeValue: "1663",
|
||||
noLeak: []string{"log_id", "body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "api error without a code is generic",
|
||||
err: errors.New("feishu api error: status=502 body=bad gateway"),
|
||||
wantCode: "feishu_api_error_generic",
|
||||
},
|
||||
{
|
||||
name: "auth failure is actionable, not a retry",
|
||||
err: errors.New("feishu auth error: code=99991663 msg=Invalid access token"),
|
||||
wantCode: "feishu_auth_or_permission",
|
||||
noLeak: []string{"Invalid access token", "99991663"},
|
||||
},
|
||||
{
|
||||
name: "unknown error falls back to sync_failed",
|
||||
err: errors.New("some totally unexpected failure"),
|
||||
wantCode: "sync_failed",
|
||||
noLeak: []string{"totally unexpected"},
|
||||
},
|
||||
{
|
||||
// "decode"/"encode"/"unicode" all contain the substring "code" but
|
||||
// are not Feishu API errors — they must not be classified as one.
|
||||
name: "decode error is not a feishu api error",
|
||||
err: errors.New("failed to decode response body"),
|
||||
wantCode: "sync_failed",
|
||||
},
|
||||
{
|
||||
// The failed item is retried on the next sync (the cursor is not
|
||||
// advanced), which for a manual-only source is not automatic. The
|
||||
// message must not promise an unconditional automatic retry.
|
||||
name: "transient message promises retry on next sync, not automatic",
|
||||
err: errors.New("feishu rate limited: status=429"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"next sync"},
|
||||
noLeak: []string{"automatically"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, codeValue, fallback := feishuFailure(tc.err)
|
||||
if code != tc.wantCode {
|
||||
t.Errorf("code = %q, want %q", code, tc.wantCode)
|
||||
}
|
||||
if codeValue != tc.wantCodeValue {
|
||||
t.Errorf("codeValue = %q, want %q", codeValue, tc.wantCodeValue)
|
||||
}
|
||||
if strings.TrimSpace(fallback) == "" {
|
||||
t.Errorf("fallback message must not be empty")
|
||||
}
|
||||
for _, s := range tc.fallbackHas {
|
||||
if !strings.Contains(strings.ToLower(fallback), s) {
|
||||
t.Errorf("fallback %q missing %q", fallback, s)
|
||||
}
|
||||
}
|
||||
for _, s := range tc.noLeak {
|
||||
if strings.Contains(code, s) || strings.Contains(fallback, s) {
|
||||
t.Errorf("raw detail %q leaked into code/fallback (code=%q fallback=%q)", s, code, fallback)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
//go:build feishu_integration
|
||||
|
||||
// Real-API integration test for FetchStream against a live Feishu tenant.
|
||||
//
|
||||
// It is OPT-IN and never runs in the normal suite or CI: it is guarded by the
|
||||
// `feishu_integration` build tag AND skips unless the required env vars are set.
|
||||
//
|
||||
// Run it with:
|
||||
//
|
||||
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
|
||||
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... \
|
||||
// go test -tags feishu_integration -run TestRealAPI -v \
|
||||
// ./internal/datasource/connector/feishu/
|
||||
//
|
||||
// The test is strictly READ-ONLY: it lists and exports documents from the given
|
||||
// wiki space but never creates, edits or deletes anything. It whitelists
|
||||
// *.feishu.cn / *.larksuite.com for SSRF in-process so it works from a dev
|
||||
// machine behind a fake-ip proxy without any production code change.
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// collectHandler records everything a real FetchStream emits so the test can
|
||||
// assert coverage and resume behaviour. cancelAfter>0 simulates a task timeout
|
||||
// by cancelling the run after that many successful content emits.
|
||||
type collectHandler struct {
|
||||
ingested []string
|
||||
failed []string
|
||||
checkpoints []*types.SyncCursor
|
||||
calls int
|
||||
cancelAfter int
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (h *collectHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if item.Metadata["error"] != "" {
|
||||
h.failed = append(h.failed, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
h.calls++
|
||||
if h.cancelAfter > 0 && h.calls > h.cancelAfter {
|
||||
if h.cancel != nil {
|
||||
h.cancel()
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
h.ingested = append(h.ingested, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *collectHandler) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
|
||||
// Snapshot the cursor JSON (mirrors the service serialising synchronously).
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
var m map[string]interface{}
|
||||
_ = json.Unmarshal(b, &m)
|
||||
h.checkpoints = append(h.checkpoints, &types.SyncCursor{ConnectorCursor: m})
|
||||
return nil
|
||||
}
|
||||
|
||||
func nodeTimes(t *testing.T, cur *types.SyncCursor) map[string]map[string]string {
|
||||
t.Helper()
|
||||
if cur == nil {
|
||||
return nil
|
||||
}
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cur.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
return fc.SpaceNodeTimes
|
||||
}
|
||||
|
||||
func countNodes(m map[string]map[string]string) int {
|
||||
n := 0
|
||||
for _, v := range m {
|
||||
n += len(v)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestRealAPI_ListSpaces discovers which wiki spaces the app can access, so the
|
||||
// space ID for the other tests can be found without guessing. Read-only.
|
||||
func TestRealAPI_ListSpaces(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
if appID == "" || appSecret == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
|
||||
}
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
|
||||
spaces, err := client.ListWikiSpaces(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListWikiSpaces: %v", err)
|
||||
}
|
||||
if len(spaces) == 0 {
|
||||
t.Fatalf("app can access 0 wiki spaces — add the app as a member of a knowledge base")
|
||||
}
|
||||
for _, s := range spaces {
|
||||
t.Logf("SPACE space_id=%s name=%q", s.SpaceID, s.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealAPI_FetchStreamResumeConverges(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
|
||||
if appID == "" || appSecret == "" || spaceID == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
|
||||
}
|
||||
|
||||
// Whitelist the Feishu/Lark hosts for SSRF so the dev-machine fake-ip proxy
|
||||
// (198.18.x) is allowed. No production code change; scoped to these hosts.
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
baseURL := os.Getenv("FEISHU_BASE_URL") // optional; defaults to region base
|
||||
cfg := &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeFeishu,
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": appID,
|
||||
"app_secret": appSecret,
|
||||
"base_url": baseURL,
|
||||
},
|
||||
ResourceIDs: []string{spaceID},
|
||||
}
|
||||
c := NewConnector(RegionFeishu)
|
||||
|
||||
// ---- Pass 1: full sync against the real space.
|
||||
h1 := &collectHandler{}
|
||||
cur1, err := c.FetchStream(context.Background(), cfg, nil, h1)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 1 (full) FetchStream error: %v", err)
|
||||
}
|
||||
n := len(h1.ingested)
|
||||
t.Logf("pass 1: ingested=%d failed=%d cursor_nodes=%d", n, len(h1.failed), countNodes(nodeTimes(t, cur1)))
|
||||
if n == 0 {
|
||||
t.Fatalf("pass 1 ingested 0 documents — create a few docs in the space, or check app scopes (wiki/docx/drive read)")
|
||||
}
|
||||
// Every ingested node must be recorded in the cursor.
|
||||
nt1 := nodeTimes(t, cur1)[spaceID]
|
||||
for _, id := range h1.ingested {
|
||||
if _, ok := nt1[id]; !ok {
|
||||
t.Errorf("pass 1 cursor missing edit time for ingested node %s", id)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Pass 2: incremental with pass-1 cursor. Against the REAL API this is
|
||||
// the load-bearing check a fake can't make: real obj_edit_time must be
|
||||
// stable for unchanged docs, so NOTHING is re-ingested.
|
||||
h2 := &collectHandler{}
|
||||
cur2, err := c.FetchStream(context.Background(), cfg, cur1, h2)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 2 (incremental) FetchStream error: %v", err)
|
||||
}
|
||||
t.Logf("pass 2: re-ingested=%d (want 0)", len(h2.ingested))
|
||||
if len(h2.ingested) != 0 {
|
||||
t.Errorf("pass 2 re-ingested %v — real edit-time is unstable or skip logic is broken", h2.ingested)
|
||||
}
|
||||
if got := countNodes(nodeTimes(t, cur2)); got != countNodes(nt1AsMap(nt1)) {
|
||||
t.Logf("note: cursor node count changed between passes (%d→%d) — space may have been edited concurrently", countNodes(nt1AsMap(nt1)), got)
|
||||
}
|
||||
|
||||
// ---- Pass 3: interrupt mid-sync then resume, against the real API.
|
||||
// Only meaningful with at least 2 documents.
|
||||
if n < 2 {
|
||||
t.Logf("only %d doc(s); skipping interrupt/resume convergence sub-test (needs >=2)", n)
|
||||
return
|
||||
}
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1 // checkpoint every node so resume is precise
|
||||
defer func() { feishuStreamCheckpointInterval = prevN }()
|
||||
|
||||
ctx3, cancel3 := context.WithCancel(context.Background())
|
||||
h3 := &collectHandler{cancelAfter: 1, cancel: cancel3} // abort after 1 success
|
||||
_, err = c.FetchStream(ctx3, cfg, nil, h3)
|
||||
cancel3()
|
||||
if err == nil {
|
||||
t.Fatalf("pass 3 expected an abort error from the simulated timeout")
|
||||
}
|
||||
if len(h3.checkpoints) == 0 {
|
||||
t.Fatalf("pass 3 wrote no checkpoint — resume would restart from scratch")
|
||||
}
|
||||
persisted := h3.checkpoints[len(h3.checkpoints)-1]
|
||||
t.Logf("pass 3: ingested=%d before abort, persisted cursor nodes=%d", len(h3.ingested), countNodes(nodeTimes(t, persisted)))
|
||||
|
||||
// Resume from the persisted checkpoint; must converge to full coverage.
|
||||
h4 := &collectHandler{}
|
||||
_, err = c.FetchStream(context.Background(), cfg, persisted, h4)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 4 (resume) FetchStream error: %v", err)
|
||||
}
|
||||
union := map[string]bool{}
|
||||
for _, id := range h3.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, id := range h4.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
t.Logf("pass 3+4 union ingested=%d, pass-1 full=%d", len(union), n)
|
||||
if len(union) < n {
|
||||
t.Errorf("resume converged to %d docs but full sync had %d — a document was lost across interrupt+resume", len(union), n)
|
||||
}
|
||||
// Resume must not redo already-checkpointed work.
|
||||
done := nodeTimes(t, persisted)[spaceID]
|
||||
for _, id := range h4.ingested {
|
||||
if _, ok := done[id]; ok {
|
||||
t.Errorf("pass 4 re-ingested already-checkpointed node %s (redundant re-export)", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func nt1AsMap(m map[string]string) map[string]map[string]string {
|
||||
return map[string]map[string]string{"_": m}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//go:build feishu_integration
|
||||
|
||||
// Seeder for the real-API integration test. OPT-IN (feishu_integration tag +
|
||||
// env vars) so it never runs in the normal suite or CI.
|
||||
//
|
||||
// It WRITES to your Feishu tenant: it creates a handful of docx documents under
|
||||
// the given wiki space, each titled "WeKnora sync test doc N" so you can find
|
||||
// and delete them afterwards. Requires app write scopes:
|
||||
// - wiki:wiki (create wiki nodes)
|
||||
// - docx:document (write docx content)
|
||||
// plus the app being a member of the space.
|
||||
//
|
||||
// Run:
|
||||
//
|
||||
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
|
||||
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... FEISHU_SEED_COUNT=3 \
|
||||
// go test -tags feishu_integration -run TestRealAPI_SeedDocs -v \
|
||||
// ./internal/datasource/connector/feishu/
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// wikiNodeCreateResp is the subset of POST /wiki/v2/spaces/:id/nodes we need.
|
||||
type wikiNodeCreateResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Node struct {
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"`
|
||||
ObjType string `json:"obj_type"`
|
||||
Title string `json:"title"`
|
||||
} `json:"node"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// TestRealAPI_CreateSpace creates a fresh knowledge base owned by the app, so
|
||||
// the app can seed and read it without a human adding it to an existing space.
|
||||
// It prints the new space_id for use as FEISHU_TEST_SPACE_ID.
|
||||
func TestRealAPI_CreateSpace(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
if appID == "" || appSecret == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
|
||||
}
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
|
||||
|
||||
var resp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Space struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"space"`
|
||||
} `json:"data"`
|
||||
}
|
||||
err := client.doRequest(context.Background(), "POST", "/open-apis/wiki/v2/spaces",
|
||||
map[string]interface{}{
|
||||
"name": "WeKnora Resilience Test KB",
|
||||
"description": "Auto-created for connector resilience testing (#2136). Safe to delete.",
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
t.Fatalf("create space: %v", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
t.Fatalf("create space: code=%d msg=%s (needs wiki:wiki write scope)", resp.Code, resp.Msg)
|
||||
}
|
||||
t.Logf("CREATED_SPACE space_id=%s name=%q", resp.Data.Space.SpaceID, resp.Data.Space.Name)
|
||||
}
|
||||
|
||||
func TestRealAPI_SeedDocs(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
|
||||
if appID == "" || appSecret == "" || spaceID == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
|
||||
}
|
||||
count := 3
|
||||
if v := os.Getenv("FEISHU_SEED_COUNT"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
count = n
|
||||
}
|
||||
}
|
||||
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
baseURL := os.Getenv("FEISHU_BASE_URL")
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: baseURL})
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 1; i <= count; i++ {
|
||||
title := fmt.Sprintf("WeKnora sync test doc %d", i)
|
||||
|
||||
// 1) Create a docx wiki node.
|
||||
var created wikiNodeCreateResp
|
||||
err := client.doRequest(ctx, "POST",
|
||||
fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes", spaceID),
|
||||
map[string]interface{}{
|
||||
"obj_type": "docx",
|
||||
"node_type": "origin",
|
||||
"title": title,
|
||||
}, &created)
|
||||
if err != nil {
|
||||
t.Fatalf("create wiki node %d: %v", i, err)
|
||||
}
|
||||
if created.Code != 0 {
|
||||
t.Fatalf("create wiki node %d: code=%d msg=%s (check wiki:wiki write scope + space membership)", i, created.Code, created.Msg)
|
||||
}
|
||||
docID := created.Data.Node.ObjToken // for docx, obj_token == document_id
|
||||
|
||||
// 2) Add a text block so the export is non-empty.
|
||||
var blockResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
berr := client.doRequest(ctx, "POST",
|
||||
fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s/children", docID, docID),
|
||||
map[string]interface{}{
|
||||
"index": 0,
|
||||
"children": []map[string]interface{}{{
|
||||
"block_type": 2, // text
|
||||
"text": map[string]interface{}{
|
||||
"elements": []map[string]interface{}{{
|
||||
"text_run": map[string]interface{}{
|
||||
"content": fmt.Sprintf("This is WeKnora sync test document number %d. It exists to validate connector resilience (issue #2136).", i),
|
||||
},
|
||||
}},
|
||||
},
|
||||
}},
|
||||
}, &blockResp)
|
||||
if berr != nil {
|
||||
t.Logf("warn: add content to doc %d failed (%v) — doc still created, export may be empty", i, berr)
|
||||
} else if blockResp.Code != 0 {
|
||||
t.Logf("warn: add content to doc %d: code=%d msg=%s (needs docx:document write scope) — doc still created", i, blockResp.Code, blockResp.Msg)
|
||||
}
|
||||
|
||||
t.Logf("created doc %d: title=%q node_token=%s obj_token=%s", i, title, created.Data.Node.NodeToken, docID)
|
||||
}
|
||||
t.Logf("seeded %d docs under space %s. Now run TestRealAPI_FetchStreamResumeConverges.", count, spaceID)
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// fakeFeishuFailingExport serves the auth/spaces/nodes endpoints normally but
|
||||
// fails every document export (code != 0), so fetchNodeContent returns an error
|
||||
// for each supported node — modelling a rate-limited / broken export.
|
||||
func fakeFeishuFailingExport(nodes []wikiNode) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiSpaceListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
|
||||
})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: nodes},
|
||||
})
|
||||
})
|
||||
// Export creation fails for every document.
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, apiResponse{Code: 1, Msg: "export unavailable"})
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}
|
||||
}
|
||||
|
||||
// A node whose fetch fails must NOT have its new edit time recorded in the
|
||||
// returned cursor: recording it would make the next sync's unchanged fast-path
|
||||
// skip it forever, silently dropping a document on a transient export failure
|
||||
// (Tencent/WeKnora#2136). With a prior edit time known, the prior value is
|
||||
// retained so prev != current next run and the node is retried.
|
||||
func TestFetchStream_FailedFetchRetainsPriorCursor(t *testing.T) {
|
||||
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
|
||||
ts, cfg := fakeFeishuFailingExport(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
cursor := makeStreamCursor(t, map[string]map[string]string{"space1": {"nt1": "50"}}) // prior, older
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
// A failure item must be surfaced, not silently dropped.
|
||||
if len(h.emitted) != 1 || h.emitted[0].Metadata["error"] == "" {
|
||||
t.Fatalf("expected 1 emitted failure item with error metadata, got %+v", h.emitted)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
got := fc.SpaceNodeTimes["space1"]["nt1"]
|
||||
if got == "100" {
|
||||
t.Fatalf("failed node advanced to current edit time %q — it will be skipped forever", got)
|
||||
}
|
||||
if got != "50" {
|
||||
t.Errorf("failed node cursor = %q, want prior value \"50\" (retry next run)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no prior cursor entry, a failed fetch must leave the node out of the
|
||||
// returned cursor entirely, so the next run treats it as new and retries it.
|
||||
func TestFetchStream_FailedFetchNoPriorOmitsFromCursor(t *testing.T) {
|
||||
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
|
||||
ts, cfg := fakeFeishuFailingExport(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
if v, ok := fc.SpaceNodeTimes["space1"]["nt1"]; ok {
|
||||
t.Errorf("failed node recorded in cursor as %q; want absent so it is retried next run", v)
|
||||
}
|
||||
}
|
||||
|
||||
// recordingHandler captures the items and checkpoints a streaming fetch emits.
|
||||
// Checkpoints are snapshotted (JSON-encoded) at call time — mirroring the
|
||||
// service, which serializes the cursor synchronously inside Checkpoint — so the
|
||||
// test observes the cursor state as it was when Checkpoint was called, not the
|
||||
// connector's later-mutated map.
|
||||
type recordingHandler struct {
|
||||
emitted []types.FetchedItem
|
||||
checkpoints []feishuCursor
|
||||
emitErr func(item types.FetchedItem) error
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if h.emitErr != nil {
|
||||
if err := h.emitErr(item); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
h.emitted = append(h.emitted, item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
h.checkpoints = append(h.checkpoints, fc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeStreamCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) *types.SyncCursor {
|
||||
t.Helper()
|
||||
prev := feishuCursor{SpaceNodeTimes: spaceNodeTimes}
|
||||
b, _ := json.Marshal(prev)
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatalf("build cursor: %v", err)
|
||||
}
|
||||
return &types.SyncCursor{ConnectorCursor: m}
|
||||
}
|
||||
|
||||
// A nil-cursor stream emits every supported node once and skips unsupported
|
||||
// types, and the returned cursor records the edit time of every discovered node
|
||||
// (so the next incremental run can detect changes).
|
||||
func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "mindnote", Title: "Brain", ObjEditTime: "200"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.emitted) != 2 {
|
||||
t.Fatalf("emitted %d items, want 2 (nt1, nt3)", len(h.emitted))
|
||||
}
|
||||
if h.emitted[0].ExternalID != "nt1" || h.emitted[1].ExternalID != "nt3" {
|
||||
t.Errorf("emitted ids = %q,%q; want nt1,nt3", h.emitted[0].ExternalID, h.emitted[1].ExternalID)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
times := fc.SpaceNodeTimes["space1"]
|
||||
for _, tok := range []string{"nt1", "nt2", "nt3"} {
|
||||
if _, ok := times[tok]; !ok {
|
||||
t.Errorf("returned cursor missing edit time for %s", tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// When a cursor already records a node at its current edit time, that node is
|
||||
// unchanged and must not be re-emitted; only new/changed nodes stream through.
|
||||
// This is the resume/incremental-skip behavior that lets a timed-out sync
|
||||
// converge across retries instead of re-exporting everything.
|
||||
func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
cursor := makeStreamCursor(t, map[string]map[string]string{
|
||||
"space1": {"nt1": "100"}, // nt1 unchanged; nt3 unknown → new
|
||||
})
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.emitted) != 1 {
|
||||
t.Fatalf("emitted %d items, want 1 (only changed nt3)", len(h.emitted))
|
||||
}
|
||||
if h.emitted[0].ExternalID != "nt3" {
|
||||
t.Errorf("emitted id = %q, want nt3", h.emitted[0].ExternalID)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoints must persist progress at page boundaries so a crash mid-sync
|
||||
// resumes from the last checkpoint. With the interval set to 1, each emitted
|
||||
// item triggers a checkpoint, and the first checkpoint must already contain the
|
||||
// first node's edit time.
|
||||
func TestFetchStream_CheckpointsProgress(t *testing.T) {
|
||||
prev := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1
|
||||
defer func() { feishuStreamCheckpointInterval = prev }()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.checkpoints) == 0 {
|
||||
t.Fatalf("expected at least one checkpoint")
|
||||
}
|
||||
first := h.checkpoints[0]
|
||||
if _, ok := first.SpaceNodeTimes["space1"]["nt1"]; !ok {
|
||||
t.Errorf("first checkpoint missing nt1 progress: %+v", first.SpaceNodeTimes)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoints must ALSO fire on elapsed time, not only every N nodes.
|
||||
// Otherwise a sync with fewer than the node interval of slow (rate-limited)
|
||||
// exports reaches the 2h task timeout having never checkpointed, and resumes
|
||||
// from scratch forever — exactly the #2136 "never fully syncs" case. With the
|
||||
// node interval effectively disabled and the time interval at 0, every
|
||||
// processed node must still produce a checkpoint.
|
||||
func TestFetchStream_CheckpointsOnElapsedTime(t *testing.T) {
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
prevT := feishuStreamCheckpointMaxInterval
|
||||
feishuStreamCheckpointInterval = 1 << 30 // never fires by count
|
||||
feishuStreamCheckpointMaxInterval = 0 // fires by elapsed time every node
|
||||
defer func() {
|
||||
feishuStreamCheckpointInterval = prevN
|
||||
feishuStreamCheckpointMaxInterval = prevT
|
||||
}()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
if len(h.checkpoints) == 0 {
|
||||
t.Fatalf("expected time-based checkpoints even though node interval never fires")
|
||||
}
|
||||
}
|
||||
|
||||
// An Emit error aborts the stream immediately — the connector must return the
|
||||
// error and stop fetching further nodes (the sync is failing; do not burn API
|
||||
// budget on the rest of the tree).
|
||||
func TestFetchStream_EmitErrorAborts(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
boom := errors.New("ingest failed")
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{emitErr: func(item types.FetchedItem) error { return boom }}
|
||||
_, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if !errors.Is(err, boom) {
|
||||
t.Fatalf("FetchStream() error = %v, want %v", err, boom)
|
||||
}
|
||||
if len(h.emitted) != 0 {
|
||||
t.Errorf("emitted %d items, want 0 (aborted on first emit)", len(h.emitted))
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -12,8 +13,9 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -775,6 +777,39 @@ func TestFetchAll_MixedTypes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger.SetOutput(&buf)
|
||||
defer logger.SetOutput(os.Stderr)
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
if _, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}); err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
out := buf.String()
|
||||
for _, want := range []string{
|
||||
"sync summary",
|
||||
"discovered=3",
|
||||
"fetched=1",
|
||||
"skipped_unsupported=2",
|
||||
"mindnote:1",
|
||||
"slides:1",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("summary log missing %q; got:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchTally_CountsAndSummary(t *testing.T) {
|
||||
tally := newFetchTally(13)
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.skip("mindnote")
|
||||
tally.skip("mindnote")
|
||||
tally.skip("slides")
|
||||
tally.fail()
|
||||
|
||||
if got := tally.skipped(); got != 3 {
|
||||
t.Errorf("skipped() = %d, want 3", got)
|
||||
}
|
||||
|
||||
summary := tally.summary()
|
||||
for _, want := range []string{
|
||||
"discovered=13",
|
||||
"fetched=3",
|
||||
"failed=1",
|
||||
"skipped_unsupported=3",
|
||||
"mindnote:2",
|
||||
"slides:1",
|
||||
} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Errorf("summary() = %q, missing %q", summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTally_EmptyHasNoSkips(t *testing.T) {
|
||||
tally := newFetchTally(0)
|
||||
if got := tally.skipped(); got != 0 {
|
||||
t.Errorf("skipped() = %d, want 0", got)
|
||||
}
|
||||
if !strings.Contains(tally.summary(), "skipped_unsupported=0") {
|
||||
t.Errorf("summary() = %q, want skipped_unsupported=0", tally.summary())
|
||||
}
|
||||
}
|
||||
@@ -358,13 +358,60 @@ type SyncResult struct {
|
||||
// Items that failed
|
||||
Failed int `json:"failed"`
|
||||
|
||||
// Detailed error messages
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
// Per-item failure samples (capped), shown in the sync-log UI.
|
||||
Errors []SyncItemError `json:"errors,omitempty"`
|
||||
|
||||
// Updated cursor for next incremental sync
|
||||
NextCursor *SyncCursor `json:"next_cursor,omitempty"`
|
||||
}
|
||||
|
||||
// SyncItemError is one user-facing failure sample. It carries a stable i18n
|
||||
// Code (+ interpolation Params) so the frontend localises it to the viewer's
|
||||
// language, plus a Message fallback for clients without the key. The raw API
|
||||
// status/body/log_id is never stored here — that stays in the server logs.
|
||||
type SyncItemError struct {
|
||||
// Title is the document title (user content, not translated).
|
||||
Title string `json:"title,omitempty"`
|
||||
// Code is a stable key the frontend maps to a localized string, e.g.
|
||||
// "feishu_rate_limited" → datasource.syncError.feishu_rate_limited.
|
||||
Code string `json:"code,omitempty"`
|
||||
// Params are interpolation values for the localized string, e.g. {"code":"1663"}.
|
||||
Params map[string]string `json:"params,omitempty"`
|
||||
// Message is a human fallback used when the client has no i18n key for Code.
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// Display renders the sample as a single plain string for server-side use
|
||||
// (logs, fatal-error detail). The localised UI string is built on the frontend
|
||||
// from Code/Params; this is only a non-localised fallback.
|
||||
func (e SyncItemError) Display() string {
|
||||
switch {
|
||||
case e.Title != "" && e.Message != "":
|
||||
return e.Title + ": " + e.Message
|
||||
case e.Message != "":
|
||||
return e.Message
|
||||
default:
|
||||
return e.Title
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarshalJSON keeps old sync logs readable: historically each error was a
|
||||
// plain JSON string, so a bare string decodes into Message.
|
||||
func (e *SyncItemError) UnmarshalJSON(b []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(b, &s); err == nil {
|
||||
e.Message = s
|
||||
return nil
|
||||
}
|
||||
type alias SyncItemError
|
||||
var a alias
|
||||
if err := json.Unmarshal(b, &a); err != nil {
|
||||
return err
|
||||
}
|
||||
*e = SyncItemError(a)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DataSourceSyncPayload represents the asynq task payload for data source sync
|
||||
type DataSourceSyncPayload struct {
|
||||
TracingContext
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// SyncResult.Errors is persisted as jsonb and read back when the sync-log drawer
|
||||
// is opened. Historically each entry was a plain string; it is now a structured
|
||||
// SyncItemError (title + i18n code + params + fallback message) so the frontend
|
||||
// can localise it. Old rows must still deserialize, so a bare string decodes into
|
||||
// the Message field.
|
||||
func TestSyncItemError_UnmarshalAcceptsLegacyStringAndObject(t *testing.T) {
|
||||
// Legacy format: array of plain strings.
|
||||
var legacy []SyncItemError
|
||||
if err := json.Unmarshal([]byte(`["季度报告: export failed"]`), &legacy); err != nil {
|
||||
t.Fatalf("legacy string form must still decode: %v", err)
|
||||
}
|
||||
if len(legacy) != 1 || legacy[0].Message != "季度报告: export failed" {
|
||||
t.Fatalf("legacy string should map to Message, got %+v", legacy)
|
||||
}
|
||||
if legacy[0].Code != "" {
|
||||
t.Errorf("legacy string must not invent a code, got %q", legacy[0].Code)
|
||||
}
|
||||
|
||||
// New format: structured object round-trips.
|
||||
in := SyncItemError{
|
||||
Title: "季度报告",
|
||||
Code: "feishu_api_error",
|
||||
Params: map[string]string{"code": "1663"},
|
||||
Message: "Feishu API error (code=1663); will retry",
|
||||
}
|
||||
b, err := json.Marshal([]SyncItemError{in})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var out []SyncItemError
|
||||
if err := json.Unmarshal(b, &out); err != nil {
|
||||
t.Fatalf("unmarshal structured: %v", err)
|
||||
}
|
||||
if len(out) != 1 || out[0].Code != "feishu_api_error" || out[0].Params["code"] != "1663" || out[0].Title != "季度报告" {
|
||||
t.Fatalf("structured form did not round-trip: %+v", out)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user