fix(feishu): resilient large-wiki sync — retry, streaming resume, failure visibility
Large Feishu wikis stopped syncing after the first batch (Tencent/WeKnora#2136: 5000+ docs, only ~150 ingested; #1262: a child-list failure aborted the run). Three compounding causes, fixed in layers: 1. Rate-limit / transient failures had no retry. Feishu's wiki/drive/export APIs are aggressively rate limited; a thousand-doc sync issues tens of thousands of calls, and a single 429/5xx burst failed whole swathes of docs. Add exponential backoff honouring Retry-After to doRequest (JSON) and downloadRawBytes (files): 429 up to 3x, 5xx once, transport errors back off. 2. The batch path fetched the entire tree into memory before ingesting, so a sync that hit the 2h task timeout lost all progress and restarted from scratch, and a transient per-document export failure advanced the cursor and skipped that doc permanently. Add an optional StreamingConnector interface, implemented for Feishu: FetchStream interleaves fetch -> ingest -> checkpoint, bounding memory to one item and persisting a per-node edit-time cursor at page boundaries. The service dispatches to it via type assertion; connectors without it fall back to FetchAll / FetchIncremental unchanged (non-breaking). A resumed/incremental run skips nodes already recorded at their current edit time (converges), a failed fetch does not advance the cursor (retried next run instead of dropped), and partial listing failures emit error items and carry prior edit times forward instead of aborting the sync. 3. Failures and skips were invisible. Unsupported node types (mindnote/slides/…) were skipped with no item, error or log ("13 docs synced only 3"). FetchAll now tallies discovered/fetched/failed/skipped per resource with a per-type skip breakdown and a progress line every 100 nodes. The backend already persisted per-item errors in SyncLog.result.errors but the frontend type omitted the field; declare `result` on SyncLog and render up to 50 failed documents in the sync-log drawer (#1262). i18n for zh-CN/en-US/ko-KR/ru-RU. result.errors is a jsonb column shipped in every list response, so it is capped at 100 samples server-side (recordSyncError); the exact failure total comes from the bounded items_failed counter, which the drawer uses for its count and "N more" line so a sync that fails thousands of docs stays bounded. Each sample is a structured, localisable SyncItemError instead of a raw "title: ...api error: status=500 body={json}" dump that repeated the title and leaked status/body/log_id. The connector (feishuFailure) classifies the error into a stable i18n code (+ the feishu code as a param); the service builds {title, code, params, message}; the drawer localises via vue-i18n to the viewer's language (zh/en/ko/ru), falling back to message. A custom UnmarshalJSON keeps old string-form logs readable. The banner is localised from items_failed too. Raw errors stay in the server logs. Follows the two-tier norm of Airbyte/Fivetran/Onyx: human, actionable, localised summaries in the UI; raw detail in logs. Transient errors say "will retry" (the cursor is retained); auth/permission errors point at the fix instead. Tests: client_retry_test (backoff), tally_test (accounting), connector_stream / datasource_stream (streaming dispatch + checkpoint), connector_convergence (timeout + transient-failure resume converges to full coverage; proven to catch #2136 via bug injection), datasource_result_cap (error-sample cap + structured localisable SyncItemError, no raw body), connector_error_reason (error→i18n-code classification), sync_item_error (legacy-string + structured back-compat) — all in the normal suite. connector_realapi / connector_seed are an opt-in real-Feishu harness gated by the feishu_integration build tag AND env vars, never run in CI.
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -117,58 +118,161 @@ func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) {
|
||||
return c.tokenCache, nil
|
||||
}
|
||||
|
||||
// doRequest executes an authenticated API request and decodes the JSON response.
|
||||
// Retry policy shared by doRequest (JSON API calls) and downloadRawBytes (file
|
||||
// downloads): 429 honours Retry-After, 5xx retries once, transport errors back off.
|
||||
const (
|
||||
feishuMaxRetries = 3
|
||||
feishuMax5xxRetries = 1
|
||||
feishuRetry5xxDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
|
||||
|
||||
// doRequest executes an authenticated API request and decodes the JSON response,
|
||||
// retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive
|
||||
// export/wiki APIs are aggressively rate limited, and a thousand-document sync
|
||||
// issues tens of thousands of calls; without backoff a single 429 burst used to
|
||||
// fail whole swathes of documents silently. 429 responses honour Retry-After;
|
||||
// 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx).
|
||||
func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error {
|
||||
const (
|
||||
maxRetries = feishuMaxRetries
|
||||
max5xxRetries = feishuMax5xxRetries
|
||||
retry5xxDelay = feishuRetry5xxDelay
|
||||
)
|
||||
backoff := feishuRetryBackoff
|
||||
|
||||
token, err := c.getTenantAccessToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
var bodyBytes []byte
|
||||
if body != nil {
|
||||
bodyBytes, err := json.Marshal(body)
|
||||
bodyBytes, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
var lastErr error
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read body once for logging + decoding
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
var bodyReader io.Reader
|
||||
if bodyBytes != nil {
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] %s %s", method, path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] %s %s (retry %d/%d)", method, path, attempt, maxRetries)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("execute request: %w", err)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
respBody, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read response body: %w", readErr)
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, backoff[attempt]); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] %s %s → status=%d bodyLen=%d body=%s",
|
||||
method, path, resp.StatusCode, len(respBody), truncate(string(respBody), 1000))
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), backoff[min(attempt, len(backoff)-1)])
|
||||
lastErr = fmt.Errorf("feishu rate limited: status=429 body=%s", truncate(string(respBody), 500))
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
lastErr = fmt.Errorf("feishu server error: status=%d body=%s", resp.StatusCode, truncate(string(respBody), 500))
|
||||
if attempt < max5xxRetries {
|
||||
if sErr := sleepCtx(ctx, retry5xxDelay); sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("feishu api error: status=%d body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
if err := json.Unmarshal(respBody, result); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// parseRetryAfter interprets a Retry-After header value (seconds) into a wait
|
||||
// duration, coercing 0/negative to a short delay and falling back when absent
|
||||
// or unparseable.
|
||||
func parseRetryAfter(header string, fallback time.Duration) time.Duration {
|
||||
if header == "" {
|
||||
return fallback
|
||||
}
|
||||
secs, err := strconv.ParseFloat(strings.TrimSpace(header), 64)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if secs <= 0 {
|
||||
return 100 * time.Millisecond
|
||||
}
|
||||
return time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
|
||||
// sleepCtx waits for d or until ctx is cancelled, returning ctx.Err() if the
|
||||
// context ends first so retries abort promptly on task cancellation/timeout.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return ctx.Err()
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// truncate truncates a string to maxLen and appends "..." if truncated.
|
||||
@@ -554,31 +658,83 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err
|
||||
}
|
||||
|
||||
url := c.baseURL + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
var lastErr error
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
for attempt := 0; attempt <= feishuMaxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if attempt == 0 {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s", path)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Feishu] download GET %s (retry %d/%d)", path, attempt, feishuMaxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("download request: %w", err)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
wait := parseRetryAfter(resp.Header.Get("Retry-After"), feishuRetryBackoff[min(attempt, len(feishuRetryBackoff)-1)])
|
||||
lastErr = fmt.Errorf("download rate limited: status=429 body=%s", truncate(string(body), 500))
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, wait); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 && resp.StatusCode < 600 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("download server error: status=%d body=%s", resp.StatusCode, truncate(string(body), 500))
|
||||
if attempt < feishuMax5xxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetry5xxDelay); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
logger.Errorf(ctx, "[Feishu] download GET %s → status=%d body=%s", path, resp.StatusCode, truncate(string(body), 500))
|
||||
return nil, fmt.Errorf("download failed: status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
lastErr = fmt.Errorf("read download body: %w", readErr)
|
||||
if attempt < feishuMaxRetries {
|
||||
if sErr := sleepCtx(ctx, feishuRetryBackoff[attempt]); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download body: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Feishu] download GET %s → OK, %d bytes", path, len(data))
|
||||
return data, nil
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// retryTestServer builds a server that always answers the auth-token call and
|
||||
// routes the given target path to h, so tests can drive doRequest's retry loop.
|
||||
func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
TenantAccessToken: "fake-token",
|
||||
Expire: 7200,
|
||||
})
|
||||
})
|
||||
mux.HandleFunc(target, h)
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}
|
||||
}
|
||||
|
||||
func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
// "0" is coerced to a short delay inside the client so the test stays fast.
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
return
|
||||
}
|
||||
writeJSON(w, apiResponse{Code: 0})
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
var resp apiResponse
|
||||
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_429ExhaustsRetries(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when 429s exceed the retry budget")
|
||||
}
|
||||
if attempts != 4 { // initial + 3 retries
|
||||
t.Errorf("attempts = %d, want 4 (1 + 3 retries)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_5xxRetriesOnce(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"internal error"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.doRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error after 5xx exhaustion")
|
||||
}
|
||||
if attempts != 2 { // initial + 1 retry
|
||||
t.Errorf("attempts = %d, want 2 (5xx retries exactly once)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/target", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"code":1,"msg":"bad request"}`)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil {
|
||||
t.Fatal("expected error on 400")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (non-429/5xx 4xx must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_RetriesOn429ThenSucceeds(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("payload-bytes"))
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
data, err := c.downloadRawBytes(context.Background(), "/dl")
|
||||
if err != nil {
|
||||
t.Fatalf("expected success after retry, got %v", err)
|
||||
}
|
||||
if string(data) != "payload-bytes" {
|
||||
t.Errorf("data = %q, want %q", string(data), "payload-bytes")
|
||||
}
|
||||
if attempts < 2 {
|
||||
t.Errorf("attempts = %d, want >= 2 (should retry download after 429)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadRawBytes_4xxNotRetried(t *testing.T) {
|
||||
var attempts int
|
||||
ts, cfg := retryTestServer("/dl", func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
})
|
||||
defer ts.Close()
|
||||
|
||||
c := NewClient(cfg)
|
||||
if _, err := c.downloadRawBytes(context.Background(), "/dl"); err == nil {
|
||||
t.Fatal("expected error on 403")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1 (403 must not retry)", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
fallback := 5 * time.Second
|
||||
tests := []struct {
|
||||
header string
|
||||
want time.Duration
|
||||
}{
|
||||
{"", fallback},
|
||||
{"0", 100 * time.Millisecond},
|
||||
{"-1", 100 * time.Millisecond}, // negative coerced to a short delay
|
||||
{"3", 3 * time.Second},
|
||||
{"abc", fallback}, // unparseable
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := parseRetryAfter(tt.header, fallback); got != tt.want {
|
||||
t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -27,6 +28,10 @@ func NewConnector(region Region) *Connector {
|
||||
return &Connector{region: region}
|
||||
}
|
||||
|
||||
// Feishu supports resumable streaming sync; the service prefers FetchStream over
|
||||
// FetchAll/FetchIncremental when a connector implements StreamingConnector.
|
||||
var _ datasource.StreamingConnector = (*Connector)(nil)
|
||||
|
||||
// Type returns the connector type identifier.
|
||||
func (c *Connector) Type() string {
|
||||
return c.region.ConnectorType
|
||||
@@ -34,6 +39,51 @@ func (c *Connector) Type() string {
|
||||
|
||||
const feishuWikiNodeResourceSeparator = ":"
|
||||
|
||||
// feishuStreamCheckpointInterval is how many processed nodes pass between
|
||||
// cursor checkpoints during a streaming fetch. Small enough that a timed-out
|
||||
// sync loses little work on resume, large enough that checkpoint persistence
|
||||
// (a DB write) does not dominate. Overridable in tests. See FetchStream.
|
||||
var feishuStreamCheckpointInterval = 50
|
||||
|
||||
// feishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as
|
||||
// well as node count. Without it, a sync of fewer than
|
||||
// feishuStreamCheckpointInterval very slow (rate-limited) exports could reach
|
||||
// the 2h task timeout having never checkpointed, and resume from scratch every
|
||||
// retry — the #2136 "never fully syncs" case. Overridable in tests.
|
||||
var feishuStreamCheckpointMaxInterval = 30 * time.Second
|
||||
|
||||
// fetchTally accumulates the outcome of fetching a wiki node subtree so the
|
||||
// connector can emit a single actionable summary. Without it, unsupported nodes
|
||||
// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users
|
||||
// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136).
|
||||
type fetchTally struct {
|
||||
discovered int
|
||||
fetched int
|
||||
failed int
|
||||
skippedByType map[string]int
|
||||
}
|
||||
|
||||
func newFetchTally(discovered int) *fetchTally {
|
||||
return &fetchTally{discovered: discovered, skippedByType: map[string]int{}}
|
||||
}
|
||||
|
||||
func (t *fetchTally) fetch() { t.fetched++ }
|
||||
func (t *fetchTally) fail() { t.failed++ }
|
||||
func (t *fetchTally) skip(objType string) { t.skippedByType[objType]++ }
|
||||
|
||||
func (t *fetchTally) skipped() int {
|
||||
n := 0
|
||||
for _, c := range t.skippedByType {
|
||||
n += c
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (t *fetchTally) summary() string {
|
||||
return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v",
|
||||
t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType)
|
||||
}
|
||||
|
||||
// Validate verifies that the Feishu configuration is valid by testing connectivity.
|
||||
func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error {
|
||||
feishuConfig, err := parseFeishuConfig(config, c.region)
|
||||
@@ -184,25 +234,35 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
|
||||
allItems = appendWikiNodeListFailureItems(allItems, spaceID, resourceID, partialErr.Failures)
|
||||
}
|
||||
|
||||
// Fetch content for each document node
|
||||
for _, node := range nodes {
|
||||
// Fetch content for each document node, tallying outcomes so a single
|
||||
// summary line explains where every discovered node went.
|
||||
tally := newFetchTally(len(nodes))
|
||||
for i, node := range nodes {
|
||||
item, err := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
|
||||
if err != nil {
|
||||
tally.fail()
|
||||
// Log error but continue with other nodes
|
||||
allItems = append(allItems, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
Metadata: feishuErrorItemMeta(err, nil),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if item != nil {
|
||||
tally.fetch()
|
||||
allItems = append(allItems, *item)
|
||||
} else {
|
||||
// Unsupported obj_type (mindnote/slides/…): skipped with no item.
|
||||
tally.skip(node.ObjType)
|
||||
}
|
||||
if n := i + 1; n%100 == 0 {
|
||||
logger.Infof(ctx, "[Feishu] sync progress resource=%s %d/%d (%s)",
|
||||
resourceID, n, len(nodes), tally.summary())
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[Feishu] sync summary resource=%s %s", resourceID, tally.summary())
|
||||
}
|
||||
|
||||
return allItems, nil
|
||||
@@ -293,9 +353,7 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": err.Error(),
|
||||
},
|
||||
Metadata: feishuErrorItemMeta(err, nil),
|
||||
})
|
||||
continue
|
||||
}
|
||||
@@ -334,6 +392,246 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
return changedItems, nextSyncCursor, nil
|
||||
}
|
||||
|
||||
// FetchStream performs a resumable, memory-bounded sync. It unifies the full
|
||||
// and incremental paths: with cursor == nil it fetches everything, and with a
|
||||
// cursor it skips nodes whose recorded edit time is unchanged — the same
|
||||
// mechanism that lets a sync which timed out mid-traversal resume from the last
|
||||
// checkpoint instead of restarting (Tencent/WeKnora#2136).
|
||||
//
|
||||
// Instead of accumulating every item in memory (FetchAll), it Emits each item
|
||||
// as it is fetched and Checkpoints the cursor every feishuStreamCheckpointInterval
|
||||
// processed nodes, so progress is durable across the Asynq task's 2h timeout.
|
||||
func (c *Connector) FetchStream(
|
||||
ctx context.Context, config *types.DataSourceConfig,
|
||||
cursor *types.SyncCursor, h datasource.StreamHandler,
|
||||
) (*types.SyncCursor, error) {
|
||||
feishuConfig, err := parseFeishuConfig(config, c.region)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := NewClient(feishuConfig)
|
||||
|
||||
var prevCursor feishuCursor
|
||||
if cursor != nil && cursor.ConnectorCursor != nil {
|
||||
cursorBytes, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(cursorBytes, &prevCursor)
|
||||
}
|
||||
|
||||
newCursor := feishuCursor{
|
||||
LastSyncTime: time.Now(),
|
||||
SpaceNodeTimes: make(map[string]map[string]string),
|
||||
}
|
||||
|
||||
resourceIDs := config.ResourceIDs
|
||||
if len(resourceIDs) == 0 {
|
||||
return nil, fmt.Errorf("no resource IDs (wiki space IDs or wiki node IDs) configured")
|
||||
}
|
||||
|
||||
processed := 0
|
||||
lastCheckpoint := time.Now()
|
||||
for _, resourceID := range resourceIDs {
|
||||
spaceID, nodeToken := parseWikiResourceID(resourceID)
|
||||
nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken)
|
||||
var partialErr *partialWikiNodeListError
|
||||
if err != nil {
|
||||
if !errors.As(err, &partialErr) {
|
||||
return nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err)
|
||||
}
|
||||
for _, item := range appendWikiNodeListFailureItems(nil, spaceID, resourceID, partialErr.Failures) {
|
||||
if eerr := h.Emit(ctx, item); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newCursor.SpaceNodeTimes[resourceID] = make(map[string]string)
|
||||
// On a partial listing, carry prior edit times forward so a later full
|
||||
// listing can still detect changes and deletions.
|
||||
if partialErr != nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
for tok, et := range prevTimes {
|
||||
newCursor.SpaceNodeTimes[resourceID][tok] = et
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentNodes := make(map[string]bool)
|
||||
tally := newFetchTally(len(nodes))
|
||||
for i, node := range nodes {
|
||||
currentNodes[node.NodeToken] = true
|
||||
editTimeStr := node.ObjEditTime
|
||||
if editTimeStr == "" {
|
||||
editTimeStr = node.NodeEditTime
|
||||
}
|
||||
|
||||
// Prior recorded edit time for this node, if any.
|
||||
var prevEdit string
|
||||
var hadPrev bool
|
||||
if prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
prevEdit, hadPrev = prevTimes[node.NodeToken]
|
||||
}
|
||||
}
|
||||
|
||||
// Resume/incremental fast-path: a node recorded at its current edit
|
||||
// time is unchanged (or already synced this run) — keep the record
|
||||
// and skip re-fetching.
|
||||
if hadPrev && prevEdit == editTimeStr {
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
|
||||
continue
|
||||
}
|
||||
|
||||
item, ferr := c.fetchNodeContent(ctx, client, node, spaceID, resourceID)
|
||||
if ferr != nil {
|
||||
tally.fail()
|
||||
// Do NOT advance the cursor: the content was never fetched.
|
||||
// Retain the prior edit time (if any) so prev != current next
|
||||
// run and the node is retried, instead of being permanently
|
||||
// skipped on a transient export failure (Tencent/WeKnora#2136).
|
||||
if hadPrev {
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = prevEdit
|
||||
}
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: node.NodeToken,
|
||||
Title: node.Title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: feishuErrorItemMeta(ferr, nil),
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
} else {
|
||||
// Fetched, or an unsupported obj_type (nothing to fetch): record
|
||||
// the current edit time so the node is not re-processed next run.
|
||||
newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr
|
||||
if item != nil {
|
||||
tally.fetch()
|
||||
if eerr := h.Emit(ctx, *item); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
} else {
|
||||
// Unsupported obj_type (mindnote/slides/…): no item.
|
||||
tally.skip(node.ObjType)
|
||||
}
|
||||
}
|
||||
|
||||
processed++
|
||||
if processed%feishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= feishuStreamCheckpointMaxInterval {
|
||||
if cerr := h.Checkpoint(ctx, newCursor.toSyncCursor()); cerr != nil {
|
||||
logger.Warnf(ctx, "[Feishu] stream checkpoint failed: %v", cerr)
|
||||
}
|
||||
lastCheckpoint = time.Now()
|
||||
}
|
||||
if n := i + 1; n%100 == 0 {
|
||||
logger.Infof(ctx, "[Feishu] stream progress resource=%s %d/%d (%s)",
|
||||
resourceID, n, len(nodes), tally.summary())
|
||||
}
|
||||
}
|
||||
|
||||
// Detect deleted nodes (only when the full tree was listed successfully).
|
||||
if partialErr == nil && prevCursor.SpaceNodeTimes != nil {
|
||||
if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok {
|
||||
for tok := range prevTimes {
|
||||
if !currentNodes[tok] {
|
||||
if eerr := h.Emit(ctx, types.FetchedItem{
|
||||
ExternalID: tok,
|
||||
IsDeleted: true,
|
||||
SourceResourceID: resourceID,
|
||||
}); eerr != nil {
|
||||
return nil, eerr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[Feishu] stream summary resource=%s %s", resourceID, tally.summary())
|
||||
}
|
||||
|
||||
return newCursor.toSyncCursor(), nil
|
||||
}
|
||||
|
||||
// toSyncCursor converts the connector-specific feishuCursor into the generic
|
||||
// SyncCursor persisted by the service. It marshals through JSON so the returned
|
||||
// value is a snapshot, decoupled from later mutation of the connector's maps.
|
||||
func (fc feishuCursor) toSyncCursor() *types.SyncCursor {
|
||||
m := make(map[string]interface{})
|
||||
cursorBytes, _ := json.Marshal(fc)
|
||||
_ = json.Unmarshal(cursorBytes, &m)
|
||||
return &types.SyncCursor{
|
||||
LastSyncTime: fc.LastSyncTime,
|
||||
ConnectorCursor: m,
|
||||
}
|
||||
}
|
||||
|
||||
var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`)
|
||||
|
||||
// feishuErrorCode extracts the numeric Feishu error code from a raw error string
|
||||
// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort.
|
||||
func feishuErrorCode(raw string) string {
|
||||
if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (mapped to a localized string on the frontend), an optional numeric Feishu
|
||||
// error code for interpolation, and an English fallback message for clients
|
||||
// without the i18n key. The raw status/JSON body/log_id is never returned here —
|
||||
// it stays in the server logs. Dumping it in the UI is the anti-pattern
|
||||
// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync
|
||||
// (the cursor is retained); auth/permission errors point at the fix instead.
|
||||
func feishuFailure(err error) (code, codeValue, fallback string) {
|
||||
if err == nil {
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
|
||||
switch {
|
||||
case strings.Contains(s, "auth error"),
|
||||
strings.Contains(s, "invalid access token"),
|
||||
strings.Contains(s, "permission"),
|
||||
strings.Contains(s, "forbidden"),
|
||||
strings.Contains(s, "status=403"):
|
||||
return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes"
|
||||
case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"):
|
||||
return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync"
|
||||
case strings.Contains(s, "timed out"),
|
||||
strings.Contains(s, "timeout"),
|
||||
strings.Contains(s, "deadline exceeded"):
|
||||
return "feishu_timeout", "", "Export or request timed out; will retry on the next sync"
|
||||
case strings.Contains(s, "server error"):
|
||||
return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync"
|
||||
case strings.Contains(s, "api error"),
|
||||
strings.Contains(s, "export task failed"),
|
||||
strings.Contains(s, "download failed"):
|
||||
if v := feishuErrorCode(err.Error()); v != "" {
|
||||
return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v)
|
||||
}
|
||||
return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync"
|
||||
default:
|
||||
return "sync_failed", "", "Sync failed; will retry on the next sync"
|
||||
}
|
||||
}
|
||||
|
||||
// feishuErrorItemMeta builds the metadata for a failed item: the raw error (for
|
||||
// server logs) plus the classified i18n code / param / fallback (for a
|
||||
// localisable SyncItemError in the UI), merged with any caller-supplied extras.
|
||||
func feishuErrorItemMeta(err error, extra map[string]string) map[string]string {
|
||||
code, codeValue, fallback := feishuFailure(err)
|
||||
m := map[string]string{
|
||||
"error": err.Error(),
|
||||
"error_reason_code": code,
|
||||
"error_reason": fallback,
|
||||
}
|
||||
if codeValue != "" {
|
||||
m["error_reason_code_value"] = codeValue
|
||||
}
|
||||
for k, v := range extra {
|
||||
m[k] = v
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, resourceID string, failures []wikiNodeListFailure) []types.FetchedItem {
|
||||
for _, failure := range failures {
|
||||
node := failure.Node
|
||||
@@ -345,13 +643,12 @@ func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, r
|
||||
ExternalID: node.NodeToken,
|
||||
Title: title,
|
||||
SourceResourceID: resourceID,
|
||||
Metadata: map[string]string{
|
||||
"error": failure.Err.Error(),
|
||||
Metadata: feishuErrorItemMeta(failure.Err, map[string]string{
|
||||
"channel": types.ChannelFeishu,
|
||||
"node_token": node.NodeToken,
|
||||
"space_id": spaceID,
|
||||
"failure_stage": "list_children",
|
||||
},
|
||||
}),
|
||||
})
|
||||
}
|
||||
return items
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// statefulFeishu is a fake Feishu server whose per-node export outcome can be
|
||||
// changed between sync passes. A node whose obj_token is in failTokens has its
|
||||
// export status poll return a failed job, so fetchNodeContent errors for it —
|
||||
// modelling a transient rate-limit / export failure that later recovers.
|
||||
type statefulFeishu struct {
|
||||
mu sync.Mutex
|
||||
failTokens map[string]bool
|
||||
}
|
||||
|
||||
func (s *statefulFeishu) setFail(tokens ...string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.failTokens = map[string]bool{}
|
||||
for _, t := range tokens {
|
||||
s.failTokens[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statefulFeishu) shouldFail(token string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.failTokens[token]
|
||||
}
|
||||
|
||||
func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFeishu) {
|
||||
s := &statefulFeishu{failTokens: map[string]bool{}}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiSpaceListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
|
||||
})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: nodes},
|
||||
})
|
||||
})
|
||||
// Export create: ticket == obj_token so the poll below can key on it.
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
writeJSON(w, exportTaskCreateResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct{ Ticket string `json:"ticket"` }{Ticket: body.Token},
|
||||
})
|
||||
})
|
||||
// Export status poll: /open-apis/drive/v1/export_tasks/<ticket>?token=<objToken>
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/file/") && strings.HasSuffix(r.URL.Path, "/download") {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("fake-docx-content"))
|
||||
return
|
||||
}
|
||||
token := r.URL.Query().Get("token")
|
||||
status := 0
|
||||
if s.shouldFail(token) {
|
||||
status = 3 // failed job
|
||||
}
|
||||
resp := exportTaskStatusResponse{apiResponse: apiResponse{Code: 0}}
|
||||
resp.Data.Result.FileToken = "file-" + token
|
||||
resp.Data.Result.FileName = "exported.docx"
|
||||
resp.Data.Result.JobStatus = status
|
||||
resp.Data.Result.JobErrorMsg = "rate limited"
|
||||
writeJSON(w, resp)
|
||||
})
|
||||
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}, s
|
||||
}
|
||||
|
||||
// convergenceHandler models the service side across a resumable sync:
|
||||
// - Emit ingests content items (mirroring the service's ctx check + apply),
|
||||
// records failure items separately, and can simulate a mid-tree task timeout
|
||||
// by canceling the run's context after cancelAfterCalls Emit calls.
|
||||
// - Checkpoint snapshots the connector cursor (the DB round-trip).
|
||||
type convergenceHandler struct {
|
||||
ingested []string // ExternalIDs of successfully ingested content items
|
||||
failed []string // ExternalIDs surfaced as failures (Metadata["error"])
|
||||
exportedTokens map[string]int
|
||||
checkpoints []feishuCursor
|
||||
calls int
|
||||
cancelAfterCall int // >0: cancel ctx and abort on the Nth Emit call
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (h *convergenceHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if err := ctx.Err(); err != nil { // mirror service Emit
|
||||
return err
|
||||
}
|
||||
h.calls++
|
||||
if h.cancelAfterCall > 0 && h.calls >= h.cancelAfterCall {
|
||||
if h.cancel != nil {
|
||||
h.cancel()
|
||||
}
|
||||
return context.Canceled // service returns ctx err → connector aborts
|
||||
}
|
||||
if item.Metadata["error"] != "" {
|
||||
h.failed = append(h.failed, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
h.ingested = append(h.ingested, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *convergenceHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
h.checkpoints = append(h.checkpoints, fc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func lastCheckpointCursor(h *convergenceHandler, t *testing.T) *types.SyncCursor {
|
||||
t.Helper()
|
||||
if len(h.checkpoints) == 0 {
|
||||
return nil
|
||||
}
|
||||
last := h.checkpoints[len(h.checkpoints)-1]
|
||||
b, _ := json.Marshal(last)
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatalf("rebuild persisted cursor: %v", err)
|
||||
}
|
||||
return &types.SyncCursor{ConnectorCursor: m}
|
||||
}
|
||||
|
||||
// TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure is the
|
||||
// end-to-end proof for Tencent/WeKnora#2136: a large-ish wiki that (a) hits a
|
||||
// transient per-node export failure and (b) is killed by the 2h task timeout
|
||||
// mid-traversal must, on the asynq retry, resume from the last checkpoint,
|
||||
// re-fetch only what is outstanding, retry the transiently-failed node, and end
|
||||
// with EVERY document synced exactly once — no permanent skip, no full restart,
|
||||
// no redundant re-export of already-done nodes.
|
||||
func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T) {
|
||||
// Checkpoint on every processed node so the persisted cursor is precise.
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1
|
||||
defer func() { feishuStreamCheckpointInterval = prevN }()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", ObjEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", ObjEditTime: "200"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "docx", Title: "Doc4", ObjEditTime: "400"},
|
||||
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "docx", Title: "Doc5", ObjEditTime: "500"},
|
||||
}
|
||||
ts, cfg, srv := newStatefulFeishu(nodes)
|
||||
defer ts.Close()
|
||||
cfgDS := makeConfig(cfg, []string{"space1"})
|
||||
c := NewConnector(RegionFeishu)
|
||||
|
||||
// ---- Pass 1: obj2 export fails transiently; task "times out" on the 3rd
|
||||
// Emit call (nt1 success, nt2 failure-item, then cancel as nt3 is emitted).
|
||||
srv.setFail("obj2")
|
||||
ctx1, cancel1 := context.WithCancel(context.Background())
|
||||
h1 := &convergenceHandler{cancelAfterCall: 3, cancel: cancel1}
|
||||
_, err1 := c.FetchStream(ctx1, cfgDS, nil, h1)
|
||||
cancel1()
|
||||
if err1 == nil {
|
||||
t.Fatalf("pass 1 expected a timeout/abort error, got nil")
|
||||
}
|
||||
|
||||
// Pass 1 ingested only nt1; nt2 surfaced as a failure; nt3+ never reached.
|
||||
if got := strings.Join(h1.ingested, ","); got != "nt1" {
|
||||
t.Fatalf("pass 1 ingested = %q, want just nt1", got)
|
||||
}
|
||||
|
||||
// The persisted cursor (last checkpoint) must contain nt1 but NOT the
|
||||
// aborted nt3, nor the transiently-failed nt2 — otherwise resume would skip
|
||||
// them forever.
|
||||
persisted := lastCheckpointCursor(h1, t)
|
||||
if persisted == nil {
|
||||
t.Fatalf("pass 1 wrote no checkpoint — resume would restart from scratch")
|
||||
}
|
||||
var pc feishuCursor
|
||||
pb, _ := json.Marshal(persisted.ConnectorCursor)
|
||||
_ = json.Unmarshal(pb, &pc)
|
||||
p := pc.SpaceNodeTimes["space1"]
|
||||
if _, ok := p["nt1"]; !ok {
|
||||
t.Errorf("persisted cursor missing nt1 (done work lost on resume)")
|
||||
}
|
||||
if _, ok := p["nt2"]; ok {
|
||||
t.Errorf("persisted cursor recorded transiently-failed nt2 — it would be skipped forever")
|
||||
}
|
||||
if _, ok := p["nt3"]; ok {
|
||||
t.Errorf("persisted cursor recorded aborted nt3 — it would be skipped forever")
|
||||
}
|
||||
|
||||
// ---- Pass 2: asynq retry. obj2 has recovered; run to completion resuming
|
||||
// from the persisted cursor.
|
||||
srv.setFail() // clear all failures
|
||||
h2 := &convergenceHandler{}
|
||||
next2, err2 := c.FetchStream(context.Background(), cfgDS, persisted, h2)
|
||||
if err2 != nil {
|
||||
t.Fatalf("pass 2 error: %v", err2)
|
||||
}
|
||||
|
||||
// nt1 was already done → must be skipped (not re-exported, not re-ingested).
|
||||
for _, id := range h2.ingested {
|
||||
if id == "nt1" {
|
||||
t.Errorf("pass 2 re-ingested nt1 — redundant re-export of completed work")
|
||||
}
|
||||
}
|
||||
// Everything still outstanding (nt2 transiently-failed, nt3 aborted, nt4, nt5)
|
||||
// must be ingested in pass 2.
|
||||
want := map[string]bool{"nt2": true, "nt3": true, "nt4": true, "nt5": true}
|
||||
got2 := map[string]bool{}
|
||||
for _, id := range h2.ingested {
|
||||
got2[id] = true
|
||||
}
|
||||
for id := range want {
|
||||
if !got2[id] {
|
||||
t.Errorf("pass 2 did not ingest %s (want all outstanding nodes)", id)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Convergence: union across both passes covers ALL five nodes exactly.
|
||||
union := map[string]bool{}
|
||||
for _, id := range h1.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, id := range h2.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if !union[n.NodeToken] {
|
||||
t.Errorf("node %s was NEVER synced across both passes — data loss (#2136)", n.NodeToken)
|
||||
}
|
||||
}
|
||||
if len(union) != len(nodes) {
|
||||
t.Errorf("synced %d distinct nodes, want %d", len(union), len(nodes))
|
||||
}
|
||||
|
||||
// Final cursor is a complete snapshot of every node (next incremental sync
|
||||
// starts clean).
|
||||
var fc feishuCursor
|
||||
nb, _ := json.Marshal(next2.ConnectorCursor)
|
||||
_ = json.Unmarshal(nb, &fc)
|
||||
for _, n := range nodes {
|
||||
if _, ok := fc.SpaceNodeTimes["space1"][n.NodeToken]; !ok {
|
||||
t.Errorf("final cursor missing %s — incremental sync would re-fetch it", n.NodeToken)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// feishuFailure classifies a raw connector/API error into a stable i18n code
|
||||
// (+ optional numeric feishu code) and an English fallback message. The frontend
|
||||
// localises by code; the raw status/body/log_id stays in the server logs and
|
||||
// must never appear in the code, codeValue or fallback.
|
||||
func TestFeishuFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
wantCode string
|
||||
wantCodeValue string
|
||||
fallbackHas []string
|
||||
noLeak []string // must appear in neither code nor fallback
|
||||
}{
|
||||
{
|
||||
name: "rate limited",
|
||||
err: errors.New("feishu rate limited: status=429 body={\"code\":99991400,\"msg\":\"too many request\"}"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"retry"},
|
||||
noLeak: []string{"body=", "{", "99991400"},
|
||||
},
|
||||
{
|
||||
name: "server 5xx",
|
||||
err: errors.New("feishu server error: status=500 body={\"code\":1663}"),
|
||||
wantCode: "feishu_server_unavailable",
|
||||
noLeak: []string{"body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "export timeout",
|
||||
err: errors.New("export 季度报告 (docx): export task timed out after 60s (ticket=abc)"),
|
||||
wantCode: "feishu_timeout",
|
||||
noLeak: []string{"ticket=", "abc"},
|
||||
},
|
||||
{
|
||||
name: "api error carries the feishu code as a param",
|
||||
err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"error\":{\"log_id\":\"20260\"}}"),
|
||||
wantCode: "feishu_api_error",
|
||||
wantCodeValue: "1663",
|
||||
noLeak: []string{"log_id", "body=", "{"},
|
||||
},
|
||||
{
|
||||
name: "api error without a code is generic",
|
||||
err: errors.New("feishu api error: status=502 body=bad gateway"),
|
||||
wantCode: "feishu_api_error_generic",
|
||||
},
|
||||
{
|
||||
name: "auth failure is actionable, not a retry",
|
||||
err: errors.New("feishu auth error: code=99991663 msg=Invalid access token"),
|
||||
wantCode: "feishu_auth_or_permission",
|
||||
noLeak: []string{"Invalid access token", "99991663"},
|
||||
},
|
||||
{
|
||||
name: "unknown error falls back to sync_failed",
|
||||
err: errors.New("some totally unexpected failure"),
|
||||
wantCode: "sync_failed",
|
||||
noLeak: []string{"totally unexpected"},
|
||||
},
|
||||
{
|
||||
// "decode"/"encode"/"unicode" all contain the substring "code" but
|
||||
// are not Feishu API errors — they must not be classified as one.
|
||||
name: "decode error is not a feishu api error",
|
||||
err: errors.New("failed to decode response body"),
|
||||
wantCode: "sync_failed",
|
||||
},
|
||||
{
|
||||
// The failed item is retried on the next sync (the cursor is not
|
||||
// advanced), which for a manual-only source is not automatic. The
|
||||
// message must not promise an unconditional automatic retry.
|
||||
name: "transient message promises retry on next sync, not automatic",
|
||||
err: errors.New("feishu rate limited: status=429"),
|
||||
wantCode: "feishu_rate_limited",
|
||||
fallbackHas: []string{"next sync"},
|
||||
noLeak: []string{"automatically"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, codeValue, fallback := feishuFailure(tc.err)
|
||||
if code != tc.wantCode {
|
||||
t.Errorf("code = %q, want %q", code, tc.wantCode)
|
||||
}
|
||||
if codeValue != tc.wantCodeValue {
|
||||
t.Errorf("codeValue = %q, want %q", codeValue, tc.wantCodeValue)
|
||||
}
|
||||
if strings.TrimSpace(fallback) == "" {
|
||||
t.Errorf("fallback message must not be empty")
|
||||
}
|
||||
for _, s := range tc.fallbackHas {
|
||||
if !strings.Contains(strings.ToLower(fallback), s) {
|
||||
t.Errorf("fallback %q missing %q", fallback, s)
|
||||
}
|
||||
}
|
||||
for _, s := range tc.noLeak {
|
||||
if strings.Contains(code, s) || strings.Contains(fallback, s) {
|
||||
t.Errorf("raw detail %q leaked into code/fallback (code=%q fallback=%q)", s, code, fallback)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
//go:build feishu_integration
|
||||
|
||||
// Real-API integration test for FetchStream against a live Feishu tenant.
|
||||
//
|
||||
// It is OPT-IN and never runs in the normal suite or CI: it is guarded by the
|
||||
// `feishu_integration` build tag AND skips unless the required env vars are set.
|
||||
//
|
||||
// Run it with:
|
||||
//
|
||||
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
|
||||
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... \
|
||||
// go test -tags feishu_integration -run TestRealAPI -v \
|
||||
// ./internal/datasource/connector/feishu/
|
||||
//
|
||||
// The test is strictly READ-ONLY: it lists and exports documents from the given
|
||||
// wiki space but never creates, edits or deletes anything. It whitelists
|
||||
// *.feishu.cn / *.larksuite.com for SSRF in-process so it works from a dev
|
||||
// machine behind a fake-ip proxy without any production code change.
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// collectHandler records everything a real FetchStream emits so the test can
|
||||
// assert coverage and resume behaviour. cancelAfter>0 simulates a task timeout
|
||||
// by cancelling the run after that many successful content emits.
|
||||
type collectHandler struct {
|
||||
ingested []string
|
||||
failed []string
|
||||
checkpoints []*types.SyncCursor
|
||||
calls int
|
||||
cancelAfter int
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (h *collectHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if item.Metadata["error"] != "" {
|
||||
h.failed = append(h.failed, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
h.calls++
|
||||
if h.cancelAfter > 0 && h.calls > h.cancelAfter {
|
||||
if h.cancel != nil {
|
||||
h.cancel()
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
h.ingested = append(h.ingested, item.ExternalID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *collectHandler) Checkpoint(_ context.Context, cursor *types.SyncCursor) error {
|
||||
// Snapshot the cursor JSON (mirrors the service serialising synchronously).
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
var m map[string]interface{}
|
||||
_ = json.Unmarshal(b, &m)
|
||||
h.checkpoints = append(h.checkpoints, &types.SyncCursor{ConnectorCursor: m})
|
||||
return nil
|
||||
}
|
||||
|
||||
func nodeTimes(t *testing.T, cur *types.SyncCursor) map[string]map[string]string {
|
||||
t.Helper()
|
||||
if cur == nil {
|
||||
return nil
|
||||
}
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cur.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
return fc.SpaceNodeTimes
|
||||
}
|
||||
|
||||
func countNodes(m map[string]map[string]string) int {
|
||||
n := 0
|
||||
for _, v := range m {
|
||||
n += len(v)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestRealAPI_ListSpaces discovers which wiki spaces the app can access, so the
|
||||
// space ID for the other tests can be found without guessing. Read-only.
|
||||
func TestRealAPI_ListSpaces(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
if appID == "" || appSecret == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
|
||||
}
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
|
||||
spaces, err := client.ListWikiSpaces(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListWikiSpaces: %v", err)
|
||||
}
|
||||
if len(spaces) == 0 {
|
||||
t.Fatalf("app can access 0 wiki spaces — add the app as a member of a knowledge base")
|
||||
}
|
||||
for _, s := range spaces {
|
||||
t.Logf("SPACE space_id=%s name=%q", s.SpaceID, s.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealAPI_FetchStreamResumeConverges(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
|
||||
if appID == "" || appSecret == "" || spaceID == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
|
||||
}
|
||||
|
||||
// Whitelist the Feishu/Lark hosts for SSRF so the dev-machine fake-ip proxy
|
||||
// (198.18.x) is allowed. No production code change; scoped to these hosts.
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
baseURL := os.Getenv("FEISHU_BASE_URL") // optional; defaults to region base
|
||||
cfg := &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeFeishu,
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": appID,
|
||||
"app_secret": appSecret,
|
||||
"base_url": baseURL,
|
||||
},
|
||||
ResourceIDs: []string{spaceID},
|
||||
}
|
||||
c := NewConnector(RegionFeishu)
|
||||
|
||||
// ---- Pass 1: full sync against the real space.
|
||||
h1 := &collectHandler{}
|
||||
cur1, err := c.FetchStream(context.Background(), cfg, nil, h1)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 1 (full) FetchStream error: %v", err)
|
||||
}
|
||||
n := len(h1.ingested)
|
||||
t.Logf("pass 1: ingested=%d failed=%d cursor_nodes=%d", n, len(h1.failed), countNodes(nodeTimes(t, cur1)))
|
||||
if n == 0 {
|
||||
t.Fatalf("pass 1 ingested 0 documents — create a few docs in the space, or check app scopes (wiki/docx/drive read)")
|
||||
}
|
||||
// Every ingested node must be recorded in the cursor.
|
||||
nt1 := nodeTimes(t, cur1)[spaceID]
|
||||
for _, id := range h1.ingested {
|
||||
if _, ok := nt1[id]; !ok {
|
||||
t.Errorf("pass 1 cursor missing edit time for ingested node %s", id)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Pass 2: incremental with pass-1 cursor. Against the REAL API this is
|
||||
// the load-bearing check a fake can't make: real obj_edit_time must be
|
||||
// stable for unchanged docs, so NOTHING is re-ingested.
|
||||
h2 := &collectHandler{}
|
||||
cur2, err := c.FetchStream(context.Background(), cfg, cur1, h2)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 2 (incremental) FetchStream error: %v", err)
|
||||
}
|
||||
t.Logf("pass 2: re-ingested=%d (want 0)", len(h2.ingested))
|
||||
if len(h2.ingested) != 0 {
|
||||
t.Errorf("pass 2 re-ingested %v — real edit-time is unstable or skip logic is broken", h2.ingested)
|
||||
}
|
||||
if got := countNodes(nodeTimes(t, cur2)); got != countNodes(nt1AsMap(nt1)) {
|
||||
t.Logf("note: cursor node count changed between passes (%d→%d) — space may have been edited concurrently", countNodes(nt1AsMap(nt1)), got)
|
||||
}
|
||||
|
||||
// ---- Pass 3: interrupt mid-sync then resume, against the real API.
|
||||
// Only meaningful with at least 2 documents.
|
||||
if n < 2 {
|
||||
t.Logf("only %d doc(s); skipping interrupt/resume convergence sub-test (needs >=2)", n)
|
||||
return
|
||||
}
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1 // checkpoint every node so resume is precise
|
||||
defer func() { feishuStreamCheckpointInterval = prevN }()
|
||||
|
||||
ctx3, cancel3 := context.WithCancel(context.Background())
|
||||
h3 := &collectHandler{cancelAfter: 1, cancel: cancel3} // abort after 1 success
|
||||
_, err = c.FetchStream(ctx3, cfg, nil, h3)
|
||||
cancel3()
|
||||
if err == nil {
|
||||
t.Fatalf("pass 3 expected an abort error from the simulated timeout")
|
||||
}
|
||||
if len(h3.checkpoints) == 0 {
|
||||
t.Fatalf("pass 3 wrote no checkpoint — resume would restart from scratch")
|
||||
}
|
||||
persisted := h3.checkpoints[len(h3.checkpoints)-1]
|
||||
t.Logf("pass 3: ingested=%d before abort, persisted cursor nodes=%d", len(h3.ingested), countNodes(nodeTimes(t, persisted)))
|
||||
|
||||
// Resume from the persisted checkpoint; must converge to full coverage.
|
||||
h4 := &collectHandler{}
|
||||
_, err = c.FetchStream(context.Background(), cfg, persisted, h4)
|
||||
if err != nil {
|
||||
t.Fatalf("pass 4 (resume) FetchStream error: %v", err)
|
||||
}
|
||||
union := map[string]bool{}
|
||||
for _, id := range h3.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
for _, id := range h4.ingested {
|
||||
union[id] = true
|
||||
}
|
||||
t.Logf("pass 3+4 union ingested=%d, pass-1 full=%d", len(union), n)
|
||||
if len(union) < n {
|
||||
t.Errorf("resume converged to %d docs but full sync had %d — a document was lost across interrupt+resume", len(union), n)
|
||||
}
|
||||
// Resume must not redo already-checkpointed work.
|
||||
done := nodeTimes(t, persisted)[spaceID]
|
||||
for _, id := range h4.ingested {
|
||||
if _, ok := done[id]; ok {
|
||||
t.Errorf("pass 4 re-ingested already-checkpointed node %s (redundant re-export)", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func nt1AsMap(m map[string]string) map[string]map[string]string {
|
||||
return map[string]map[string]string{"_": m}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//go:build feishu_integration
|
||||
|
||||
// Seeder for the real-API integration test. OPT-IN (feishu_integration tag +
|
||||
// env vars) so it never runs in the normal suite or CI.
|
||||
//
|
||||
// It WRITES to your Feishu tenant: it creates a handful of docx documents under
|
||||
// the given wiki space, each titled "WeKnora sync test doc N" so you can find
|
||||
// and delete them afterwards. Requires app write scopes:
|
||||
// - wiki:wiki (create wiki nodes)
|
||||
// - docx:document (write docx content)
|
||||
// plus the app being a member of the space.
|
||||
//
|
||||
// Run:
|
||||
//
|
||||
// export CPLUS_INCLUDE_PATH="$(xcrun --show-sdk-path)/usr/include/c++/v1"
|
||||
// FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... FEISHU_SEED_COUNT=3 \
|
||||
// go test -tags feishu_integration -run TestRealAPI_SeedDocs -v \
|
||||
// ./internal/datasource/connector/feishu/
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// wikiNodeCreateResp is the subset of POST /wiki/v2/spaces/:id/nodes we need.
|
||||
type wikiNodeCreateResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Node struct {
|
||||
NodeToken string `json:"node_token"`
|
||||
ObjToken string `json:"obj_token"`
|
||||
ObjType string `json:"obj_type"`
|
||||
Title string `json:"title"`
|
||||
} `json:"node"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// TestRealAPI_CreateSpace creates a fresh knowledge base owned by the app, so
|
||||
// the app can seed and read it without a human adding it to an existing space.
|
||||
// It prints the new space_id for use as FEISHU_TEST_SPACE_ID.
|
||||
func TestRealAPI_CreateSpace(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
if appID == "" || appSecret == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run")
|
||||
}
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")})
|
||||
|
||||
var resp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Space struct {
|
||||
SpaceID string `json:"space_id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"space"`
|
||||
} `json:"data"`
|
||||
}
|
||||
err := client.doRequest(context.Background(), "POST", "/open-apis/wiki/v2/spaces",
|
||||
map[string]interface{}{
|
||||
"name": "WeKnora Resilience Test KB",
|
||||
"description": "Auto-created for connector resilience testing (#2136). Safe to delete.",
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
t.Fatalf("create space: %v", err)
|
||||
}
|
||||
if resp.Code != 0 {
|
||||
t.Fatalf("create space: code=%d msg=%s (needs wiki:wiki write scope)", resp.Code, resp.Msg)
|
||||
}
|
||||
t.Logf("CREATED_SPACE space_id=%s name=%q", resp.Data.Space.SpaceID, resp.Data.Space.Name)
|
||||
}
|
||||
|
||||
func TestRealAPI_SeedDocs(t *testing.T) {
|
||||
appID := os.Getenv("FEISHU_APP_ID")
|
||||
appSecret := os.Getenv("FEISHU_APP_SECRET")
|
||||
spaceID := os.Getenv("FEISHU_TEST_SPACE_ID")
|
||||
if appID == "" || appSecret == "" || spaceID == "" {
|
||||
t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_TEST_SPACE_ID to run")
|
||||
}
|
||||
count := 3
|
||||
if v := os.Getenv("FEISHU_SEED_COUNT"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
count = n
|
||||
}
|
||||
}
|
||||
|
||||
utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com")
|
||||
|
||||
baseURL := os.Getenv("FEISHU_BASE_URL")
|
||||
client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: baseURL})
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 1; i <= count; i++ {
|
||||
title := fmt.Sprintf("WeKnora sync test doc %d", i)
|
||||
|
||||
// 1) Create a docx wiki node.
|
||||
var created wikiNodeCreateResp
|
||||
err := client.doRequest(ctx, "POST",
|
||||
fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes", spaceID),
|
||||
map[string]interface{}{
|
||||
"obj_type": "docx",
|
||||
"node_type": "origin",
|
||||
"title": title,
|
||||
}, &created)
|
||||
if err != nil {
|
||||
t.Fatalf("create wiki node %d: %v", i, err)
|
||||
}
|
||||
if created.Code != 0 {
|
||||
t.Fatalf("create wiki node %d: code=%d msg=%s (check wiki:wiki write scope + space membership)", i, created.Code, created.Msg)
|
||||
}
|
||||
docID := created.Data.Node.ObjToken // for docx, obj_token == document_id
|
||||
|
||||
// 2) Add a text block so the export is non-empty.
|
||||
var blockResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
berr := client.doRequest(ctx, "POST",
|
||||
fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s/children", docID, docID),
|
||||
map[string]interface{}{
|
||||
"index": 0,
|
||||
"children": []map[string]interface{}{{
|
||||
"block_type": 2, // text
|
||||
"text": map[string]interface{}{
|
||||
"elements": []map[string]interface{}{{
|
||||
"text_run": map[string]interface{}{
|
||||
"content": fmt.Sprintf("This is WeKnora sync test document number %d. It exists to validate connector resilience (issue #2136).", i),
|
||||
},
|
||||
}},
|
||||
},
|
||||
}},
|
||||
}, &blockResp)
|
||||
if berr != nil {
|
||||
t.Logf("warn: add content to doc %d failed (%v) — doc still created, export may be empty", i, berr)
|
||||
} else if blockResp.Code != 0 {
|
||||
t.Logf("warn: add content to doc %d: code=%d msg=%s (needs docx:document write scope) — doc still created", i, blockResp.Code, blockResp.Msg)
|
||||
}
|
||||
|
||||
t.Logf("created doc %d: title=%q node_token=%s obj_token=%s", i, title, created.Data.Node.NodeToken, docID)
|
||||
}
|
||||
t.Logf("seeded %d docs under space %s. Now run TestRealAPI_FetchStreamResumeConverges.", count, spaceID)
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// fakeFeishuFailingExport serves the auth/spaces/nodes endpoints normally but
|
||||
// fails every document export (code != 0), so fetchNodeContent returns an error
|
||||
// for each supported node — modelling a rate-limited / broken export.
|
||||
func fakeFeishuFailingExport(nodes []wikiNode) (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiSpaceListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiSpace `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}},
|
||||
})
|
||||
})
|
||||
mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, wikiNodeListResponse{
|
||||
apiResponse: apiResponse{Code: 0},
|
||||
Data: struct {
|
||||
Items []wikiNode `json:"items"`
|
||||
HasMore bool `json:"has_more"`
|
||||
PageToken string `json:"page_token"`
|
||||
}{Items: nodes},
|
||||
})
|
||||
})
|
||||
// Export creation fails for every document.
|
||||
mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, apiResponse{Code: 1, Msg: "export unavailable"})
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}
|
||||
}
|
||||
|
||||
// A node whose fetch fails must NOT have its new edit time recorded in the
|
||||
// returned cursor: recording it would make the next sync's unchanged fast-path
|
||||
// skip it forever, silently dropping a document on a transient export failure
|
||||
// (Tencent/WeKnora#2136). With a prior edit time known, the prior value is
|
||||
// retained so prev != current next run and the node is retried.
|
||||
func TestFetchStream_FailedFetchRetainsPriorCursor(t *testing.T) {
|
||||
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
|
||||
ts, cfg := fakeFeishuFailingExport(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
cursor := makeStreamCursor(t, map[string]map[string]string{"space1": {"nt1": "50"}}) // prior, older
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
// A failure item must be surfaced, not silently dropped.
|
||||
if len(h.emitted) != 1 || h.emitted[0].Metadata["error"] == "" {
|
||||
t.Fatalf("expected 1 emitted failure item with error metadata, got %+v", h.emitted)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
got := fc.SpaceNodeTimes["space1"]["nt1"]
|
||||
if got == "100" {
|
||||
t.Fatalf("failed node advanced to current edit time %q — it will be skipped forever", got)
|
||||
}
|
||||
if got != "50" {
|
||||
t.Errorf("failed node cursor = %q, want prior value \"50\" (retry next run)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no prior cursor entry, a failed fetch must leave the node out of the
|
||||
// returned cursor entirely, so the next run treats it as new and retries it.
|
||||
func TestFetchStream_FailedFetchNoPriorOmitsFromCursor(t *testing.T) {
|
||||
nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}}
|
||||
ts, cfg := fakeFeishuFailingExport(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
if v, ok := fc.SpaceNodeTimes["space1"]["nt1"]; ok {
|
||||
t.Errorf("failed node recorded in cursor as %q; want absent so it is retried next run", v)
|
||||
}
|
||||
}
|
||||
|
||||
// recordingHandler captures the items and checkpoints a streaming fetch emits.
|
||||
// Checkpoints are snapshotted (JSON-encoded) at call time — mirroring the
|
||||
// service, which serializes the cursor synchronously inside Checkpoint — so the
|
||||
// test observes the cursor state as it was when Checkpoint was called, not the
|
||||
// connector's later-mutated map.
|
||||
type recordingHandler struct {
|
||||
emitted []types.FetchedItem
|
||||
checkpoints []feishuCursor
|
||||
emitErr func(item types.FetchedItem) error
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Emit(ctx context.Context, item types.FetchedItem) error {
|
||||
if h.emitErr != nil {
|
||||
if err := h.emitErr(item); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
h.emitted = append(h.emitted, item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *recordingHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error {
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(cursor.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
h.checkpoints = append(h.checkpoints, fc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeStreamCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) *types.SyncCursor {
|
||||
t.Helper()
|
||||
prev := feishuCursor{SpaceNodeTimes: spaceNodeTimes}
|
||||
b, _ := json.Marshal(prev)
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatalf("build cursor: %v", err)
|
||||
}
|
||||
return &types.SyncCursor{ConnectorCursor: m}
|
||||
}
|
||||
|
||||
// A nil-cursor stream emits every supported node once and skips unsupported
|
||||
// types, and the returned cursor records the edit time of every discovered node
|
||||
// (so the next incremental run can detect changes).
|
||||
func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt2", ObjToken: "obj2", ObjType: "mindnote", Title: "Brain", ObjEditTime: "200"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.emitted) != 2 {
|
||||
t.Fatalf("emitted %d items, want 2 (nt1, nt3)", len(h.emitted))
|
||||
}
|
||||
if h.emitted[0].ExternalID != "nt1" || h.emitted[1].ExternalID != "nt3" {
|
||||
t.Errorf("emitted ids = %q,%q; want nt1,nt3", h.emitted[0].ExternalID, h.emitted[1].ExternalID)
|
||||
}
|
||||
|
||||
var fc feishuCursor
|
||||
b, _ := json.Marshal(next.ConnectorCursor)
|
||||
_ = json.Unmarshal(b, &fc)
|
||||
times := fc.SpaceNodeTimes["space1"]
|
||||
for _, tok := range []string{"nt1", "nt2", "nt3"} {
|
||||
if _, ok := times[tok]; !ok {
|
||||
t.Errorf("returned cursor missing edit time for %s", tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// When a cursor already records a node at its current edit time, that node is
|
||||
// unchanged and must not be re-emitted; only new/changed nodes stream through.
|
||||
// This is the resume/incremental-skip behavior that lets a timed-out sync
|
||||
// converge across retries instead of re-exporting everything.
|
||||
func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
cursor := makeStreamCursor(t, map[string]map[string]string{
|
||||
"space1": {"nt1": "100"}, // nt1 unchanged; nt3 unknown → new
|
||||
})
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.emitted) != 1 {
|
||||
t.Fatalf("emitted %d items, want 1 (only changed nt3)", len(h.emitted))
|
||||
}
|
||||
if h.emitted[0].ExternalID != "nt3" {
|
||||
t.Errorf("emitted id = %q, want nt3", h.emitted[0].ExternalID)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoints must persist progress at page boundaries so a crash mid-sync
|
||||
// resumes from the last checkpoint. With the interval set to 1, each emitted
|
||||
// item triggers a checkpoint, and the first checkpoint must already contain the
|
||||
// first node's edit time.
|
||||
func TestFetchStream_CheckpointsProgress(t *testing.T) {
|
||||
prev := feishuStreamCheckpointInterval
|
||||
feishuStreamCheckpointInterval = 1
|
||||
defer func() { feishuStreamCheckpointInterval = prev }()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
|
||||
if len(h.checkpoints) == 0 {
|
||||
t.Fatalf("expected at least one checkpoint")
|
||||
}
|
||||
first := h.checkpoints[0]
|
||||
if _, ok := first.SpaceNodeTimes["space1"]["nt1"]; !ok {
|
||||
t.Errorf("first checkpoint missing nt1 progress: %+v", first.SpaceNodeTimes)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoints must ALSO fire on elapsed time, not only every N nodes.
|
||||
// Otherwise a sync with fewer than the node interval of slow (rate-limited)
|
||||
// exports reaches the 2h task timeout having never checkpointed, and resumes
|
||||
// from scratch forever — exactly the #2136 "never fully syncs" case. With the
|
||||
// node interval effectively disabled and the time interval at 0, every
|
||||
// processed node must still produce a checkpoint.
|
||||
func TestFetchStream_CheckpointsOnElapsedTime(t *testing.T) {
|
||||
prevN := feishuStreamCheckpointInterval
|
||||
prevT := feishuStreamCheckpointMaxInterval
|
||||
feishuStreamCheckpointInterval = 1 << 30 // never fires by count
|
||||
feishuStreamCheckpointMaxInterval = 0 // fires by elapsed time every node
|
||||
defer func() {
|
||||
feishuStreamCheckpointInterval = prevN
|
||||
feishuStreamCheckpointMaxInterval = prevT
|
||||
}()
|
||||
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{}
|
||||
if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil {
|
||||
t.Fatalf("FetchStream() error: %v", err)
|
||||
}
|
||||
if len(h.checkpoints) == 0 {
|
||||
t.Fatalf("expected time-based checkpoints even though node interval never fires")
|
||||
}
|
||||
}
|
||||
|
||||
// An Emit error aborts the stream immediately — the connector must return the
|
||||
// error and stop fetching further nodes (the sync is failing; do not burn API
|
||||
// budget on the rest of the tree).
|
||||
func TestFetchStream_EmitErrorAborts(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"},
|
||||
{NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
boom := errors.New("ingest failed")
|
||||
c := NewConnector(RegionFeishu)
|
||||
h := &recordingHandler{emitErr: func(item types.FetchedItem) error { return boom }}
|
||||
_, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h)
|
||||
if !errors.Is(err, boom) {
|
||||
t.Fatalf("FetchStream() error = %v, want %v", err, boom)
|
||||
}
|
||||
if len(h.emitted) != 0 {
|
||||
t.Errorf("emitted %d items, want 0 (aborted on first emit)", len(h.emitted))
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -12,8 +13,9 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -775,6 +777,39 @@ func TestFetchAll_MixedTypes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"},
|
||||
{NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"},
|
||||
}
|
||||
ts, cfg := fakeFeishu(nodes)
|
||||
defer ts.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger.SetOutput(&buf)
|
||||
defer logger.SetOutput(os.Stderr)
|
||||
|
||||
c := NewConnector(RegionFeishu)
|
||||
if _, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}); err != nil {
|
||||
t.Fatalf("FetchAll() error: %v", err)
|
||||
}
|
||||
|
||||
out := buf.String()
|
||||
for _, want := range []string{
|
||||
"sync summary",
|
||||
"discovered=3",
|
||||
"fetched=1",
|
||||
"skipped_unsupported=2",
|
||||
"mindnote:1",
|
||||
"slides:1",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("summary log missing %q; got:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) {
|
||||
nodes := []wikiNode{
|
||||
{NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true},
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchTally_CountsAndSummary(t *testing.T) {
|
||||
tally := newFetchTally(13)
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.fetch()
|
||||
tally.skip("mindnote")
|
||||
tally.skip("mindnote")
|
||||
tally.skip("slides")
|
||||
tally.fail()
|
||||
|
||||
if got := tally.skipped(); got != 3 {
|
||||
t.Errorf("skipped() = %d, want 3", got)
|
||||
}
|
||||
|
||||
summary := tally.summary()
|
||||
for _, want := range []string{
|
||||
"discovered=13",
|
||||
"fetched=3",
|
||||
"failed=1",
|
||||
"skipped_unsupported=3",
|
||||
"mindnote:2",
|
||||
"slides:1",
|
||||
} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Errorf("summary() = %q, missing %q", summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTally_EmptyHasNoSkips(t *testing.T) {
|
||||
tally := newFetchTally(0)
|
||||
if got := tally.skipped(); got != 0 {
|
||||
t.Errorf("skipped() = %d, want 0", got)
|
||||
}
|
||||
if !strings.Contains(tally.summary(), "skipped_unsupported=0") {
|
||||
t.Errorf("summary() = %q, want skipped_unsupported=0", tally.summary())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user