Files
Enterprise-Knowledge-Base/internal/datasource/connector/feishu/client_retry_test.go
T
nullkey c695125dce fix(feishu): resilient large-wiki sync — retry, streaming resume, failure visibility
Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136:
5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run).
Three compounding causes, fixed in layers:

1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export
   APIs are aggressively rate limited; a thousand-doc sync issues tens of
   thousands of calls, and a single 429/5xx burst failed whole swathes of docs.
   Add exponential backoff honouring Retry-After to doRequest (JSON) and
   downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off.

2. The batch path fetched the entire tree into memory before ingesting, so a
   sync that hit the 2h task timeout lost all progress and restarted from
   scratch, and a transient per-document export failure advanced the cursor and
   skipped that doc permanently. Add an optional StreamingConnector interface,
   implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint,
   bounding memory to one item and persisting a per-node edit-time cursor at
   page boundaries. The service dispatches to it via type assertion; connectors
   without it fall back to FetchAll / FetchIncremental unchanged (non-breaking).
   A resumed/incremental run skips nodes already recorded at their current edit
   time (converges), a failed fetch does not advance the cursor (retried next
   run instead of dropped), and partial listing failures emit error items and
   carry prior edit times forward instead of aborting the sync.

3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…)
   were skipped with no item, error or log ("13 docs synced only 3"). FetchAll
   now tallies discovered/fetched/failed/skipped per resource with a per-type
   skip breakdown and a progress line every 100 nodes. The backend already
   persisted per-item errors in SyncLog.result.errors but the frontend type
   omitted the field; declare `result` on SyncLog and render up to 50 failed
   documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU.
   result.errors is a jsonb column shipped in every list response, so it is
   capped at 100 samples server-side (recordSyncError); the exact failure total
   comes from the bounded items_failed counter, which the drawer uses for its
   count and "N more" line so a sync that fails thousands of docs stays bounded.
   Each sample is a structured, localisable SyncItemError instead of a raw
   "title: ...api error: status=500 body={json}" dump that repeated the title
   and leaked status/body/log_id. The connector (feishuFailure) classifies the
   error into a stable i18n code (+ the feishu code as a param); the service
   builds {title, code, params, message}; the drawer localises via vue-i18n to
   the viewer's language (zh/en/ko/ru), falling back to message. A custom
   UnmarshalJSON keeps old string-form logs readable. The banner is localised
   from items_failed too. Raw errors stay in the server logs. Follows the
   two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised
   summaries in the UI; raw detail in logs. Transient errors say "will retry"
   (the cursor is retained); auth/permission errors point at the fix instead.

Tests: client_retry_test (backoff), tally_test (accounting), connector_stream /
datasource_stream (streaming dispatch + checkpoint), connector_convergence
(timeout + transient-failure resume converges to full coverage; proven to catch
#2136 via bug injection), datasource_result_cap (error-sample cap + structured
localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code
classification), sync_item_error (legacy-string + structured back-compat) — all
in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu
harness gated by the feishu_integration build tag AND env vars, never run in CI.
2026-07-21 16:14:22 +08:00

174 lines
5.0 KiB
Go

package feishu
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
)
// retryTestServer builds a server that always answers the auth-token call and
// routes the given target path to h, so tests can drive doRequest's retry loop.
func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) {
mux := http.NewServeMux()
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, tokenResponse{
apiResponse: apiResponse{Code: 0},
TenantAccessToken: "fake-token",
Expire: 7200,
})
})
mux.HandleFunc(target, h)
ts := httptest.NewServer(mux)
return ts, &Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}
}
func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
// "0" is coerced to a short delay inside the client so the test stays fast.
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
return
}
writeJSON(w, apiResponse{Code: 0})
})
defer ts.Close()
c := NewClient(cfg)
var resp apiResponse
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil {
t.Fatalf("expected success after retry, got %v", err)
}
if attempts < 2 {
t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts)
}
}
func TestDoRequest_429ExhaustsRetries(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
})
defer ts.Close()
c := NewClient(cfg)
err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil)
if err == nil {
t.Fatal("expected error when 429s exceed the retry budget")
}
if attempts != 4 { // initial + 3 retries
t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts)
}
}
func TestDoRequest_5xxRetriesOnce(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`)
})
defer ts.Close()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
c := NewClient(cfg)
if err := c.doRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil {
t.Fatal("expected error after 5xx exhaustion")
}
if attempts != 2 { // initial + 1 retry
t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts)
}
}
func TestDoRequest_4xxNotRetried(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusBadRequest)
_, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`)
})
defer ts.Close()
c := NewClient(cfg)
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil {
t.Fatal("expected error on 400")
}
if attempts != 1 {
t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts)
}
}
func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("payload-bytes"))
})
defer ts.Close()
c := NewClient(cfg)
data, err := c.downloadRawBytes(context.Background(), "/dl")
if err != nil {
t.Fatalf("expected success after retry, got %v", err)
}
if string(data) != "payload-bytes" {
t.Errorf("data = %q, want %q", string(data), "payload-bytes")
}
if attempts < 2 {
t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts)
}
}
func TestDownloadRawBytes_4xxNotRetried(t *testing.T) {
var attempts int
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
attempts++
w.WriteHeader(http.StatusForbidden)
})
defer ts.Close()
c := NewClient(cfg)
if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil {
t.Fatal("expected error on 403")
}
if attempts != 1 {
t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts)
}
}
func TestParseRetryAfter(t *testing.T) {
fallback := 5 * time.Second
tests := []struct {
header string
want time.Duration
}{
{"", fallback},
{"0", 100 * time.Millisecond},
{"-1", 100 * time.Millisecond}, // negative coerced to a short delay
{"3", 3 * time.Second},
{"abc", fallback}, // unparseable
}
for _, tt := range tests {
if got := parseRetryAfter(tt.header, fallback); got != tt.want {
t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want)
}
}
}