From 9f35abafffe165c358cbcff4c7a2c6d7a71ad1b5 Mon Sep 17 00:00:00 2001 From: nullkey Date: Mon, 6 Apr 2026 23:31:38 +0800 Subject: [PATCH] feat(notion): add Notion API client and type definitions - Define notionPage, notionBlock, notionParent, notionFile and cursor types - Implement rate-limited HTTP client with context-aware retry and backoff - Support search, page/block retrieval, database queries, and file downloads - Add HasChildren field to Resource type, NotionChannel to knowledge types - Add unit tests for types and client --- .../datasource/connector/notion/client.go | 523 ++++++++++++++++++ .../connector/notion/client_test.go | 335 +++++++++++ internal/datasource/connector/notion/types.go | 273 +++++++++ .../datasource/connector/notion/types_test.go | 144 +++++ internal/types/datasource.go | 3 + internal/types/knowledge.go | 1 + 6 files changed, 1279 insertions(+) create mode 100644 internal/datasource/connector/notion/client.go create mode 100644 internal/datasource/connector/notion/client_test.go create mode 100644 internal/datasource/connector/notion/types.go create mode 100644 internal/datasource/connector/notion/types_test.go diff --git a/internal/datasource/connector/notion/client.go b/internal/datasource/connector/notion/client.go new file mode 100644 index 00000000..a91e355e --- /dev/null +++ b/internal/datasource/connector/notion/client.go @@ -0,0 +1,523 @@ +package notion + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" +) + +// notionClient wraps the Notion API with rate limiting and retry logic. +type notionClient struct { + token string + httpClient *http.Client + limiter *rate.Limiter + baseURL string +} + +// newClient creates a new Notion API client. +func newClient(token, baseURL string) *notionClient { + if baseURL == "" { + baseURL = DefaultBaseURL + } + return ¬ionClient{ + token: token, + httpClient: &http.Client{Timeout: 30 * time.Second}, + limiter: rate.NewLimiter(rate.Limit(3), 3), // 3 req/s with burst of 3 (matches Notion's actual behavior) + baseURL: baseURL, + } +} + +const maxRetries = 3 + +// sleepWithContext pauses for the given duration, returning early if ctx is cancelled. +func sleepWithContext(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// doRequest performs an authenticated, rate-limited HTTP request to the Notion API. +func (c *notionClient) doRequest(ctx context.Context, method, path string, body interface{}) ([]byte, error) { + if err := c.limiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("rate limiter: %w", err) + } + + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bodyReader) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Notion-Version", NotionAPIVersion) + req.Header.Set("Content-Type", "application/json") + + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + if body != nil { + bodyBytes, _ := json.Marshal(body) + req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + } + } + + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = err + if attempt < maxRetries { + if sErr := sleepWithContext(ctx, time.Duration(1<= 200 && resp.StatusCode < 300: + return respBody, nil + + case resp.StatusCode == 401 || resp.StatusCode == 403: + return nil, fmt.Errorf("%w: %s", datasource.ErrInvalidCredentials, string(respBody)) + + case resp.StatusCode == 404: + return nil, fmt.Errorf("%w: %s", datasource.ErrResourceNotFound, path) + + case resp.StatusCode == 429: + retryAfter := resp.Header.Get("Retry-After") + wait := 1 * time.Second + if secs, err := strconv.ParseFloat(retryAfter, 64); err == nil && secs > 0 { + wait = time.Duration(secs * float64(time.Second)) + } + logger.Warnf(ctx, "[Notion] rate limited, retry after %v (attempt %d/%d)", wait, attempt+1, maxRetries) + lastErr = fmt.Errorf("rate limited: %s", string(respBody)) + if attempt < maxRetries { + if sErr := sleepWithContext(ctx, wait); sErr != nil { + return nil, sErr + } + continue + } + + case resp.StatusCode >= 500: + lastErr = fmt.Errorf("server error %d: %s", resp.StatusCode, string(respBody)) + if attempt < maxRetries { + if sErr := sleepWithContext(ctx, time.Duration(1< 0 { + dsID = dsResult.DataSources[0].ID + } + + return &databaseInfo{Page: db, DataSourceID: dsID}, nil +} + +// GetDataSourceInfo retrieves a data source by ID, returning its metadata. +// In API 2025-09-03+, data_source objects hold the schema/properties and are +// the target for record queries. The response includes database_parent to +// locate the database in the workspace hierarchy. +func (c *notionClient) GetDataSourceInfo(ctx context.Context, dsID string) (*notionPage, error) { + respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/data_sources/"+dsID, nil) + if err != nil { + return nil, err + } + var ds notionPage + if err := json.Unmarshal(respBody, &ds); err != nil { + return nil, fmt.Errorf("unmarshal data_source: %w", err) + } + ds.Title = extractTitle(&ds) + return &ds, nil +} + +// GetBlockChildrenFlat fetches only the direct children of a block (no recursion). +// Used by discoverPages to quickly scan for child_page/child_database without +// fetching the full block tree content. +func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string) ([]notionBlock, error) { + var allBlocks []notionBlock + var startCursor string + + for { + path := fmt.Sprintf("/v1/blocks/%s/children", blockID) + if startCursor != "" { + path += "?start_cursor=" + startCursor + } + + respBody, err := c.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("get block children for %s: %w", blockID, err) + } + + var resp paginatedResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("unmarshal block children response: %w", err) + } + + var blocks []notionBlock + if err := json.Unmarshal(resp.Results, &blocks); err != nil { + return nil, fmt.Errorf("unmarshal blocks: %w", err) + } + + allBlocks = append(allBlocks, blocks...) + + if !resp.HasMore || resp.NextCursor == "" { + break + } + startCursor = resp.NextCursor + } + + return allBlocks, nil +} + +const maxBlockDepth = 5 // Limit recursion depth — deeper content has diminishing value for knowledge bases +const maxBlocksPerPage = 1000 // Limit total blocks fetched per page to prevent runaway API calls + +// GetBlockChildrenAll recursively fetches all blocks under a given block ID, +// building a tree structure with Children populated for blocks with has_children=true. +// child_page and child_database blocks are NOT recursed into (handled by connector layer). +// Recursion is limited to maxBlockDepth to prevent excessive API calls on complex pages. +func (c *notionClient) GetBlockChildrenAll(ctx context.Context, blockID string) ([]notionBlock, error) { + return c.getBlockChildrenRecursive(ctx, blockID, 0) +} + +func (c *notionClient) getBlockChildrenRecursive(ctx context.Context, blockID string, depth int) ([]notionBlock, error) { + var allBlocks []notionBlock + var startCursor string + + for { + path := fmt.Sprintf("/v1/blocks/%s/children", blockID) + if startCursor != "" { + path += "?start_cursor=" + startCursor + } + + respBody, err := c.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("get block children for %s: %w", blockID, err) + } + + var resp paginatedResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("unmarshal block children response: %w", err) + } + + var blocks []notionBlock + if err := json.Unmarshal(resp.Results, &blocks); err != nil { + return nil, fmt.Errorf("unmarshal blocks: %w", err) + } + + allBlocks = append(allBlocks, blocks...) + + if len(allBlocks) >= maxBlocksPerPage || !resp.HasMore || resp.NextCursor == "" { + break + } + startCursor = resp.NextCursor + } + + if depth >= maxBlockDepth { + return allBlocks, nil + } + + for i := range allBlocks { + if !allBlocks[i].HasChildren { + continue + } + // Skip block types that don't contribute useful content + switch allBlocks[i].Type { + case "child_page", "child_database", "unsupported", "template", "breadcrumb", "table_of_contents": + continue + } + children, err := c.getBlockChildrenRecursive(ctx, allBlocks[i].ID, depth+1) + if err != nil { + logger.Warnf(ctx, "[Notion] failed to get children for block %s (depth %d): %v", allBlocks[i].ID, depth, err) + continue + } + allBlocks[i].Children = children + } + + return allBlocks, nil +} + +// QueryDatabaseAll retrieves all records from a database via POST /v1/data_sources/{id}/query. +// Accepts either a data_source_id (from search) or a database_id (from child_database blocks). +// For database_ids, resolves to data_source_id via GET /v1/databases/{id}. +func (c *notionClient) QueryDatabaseAll(ctx context.Context, id string) ([]notionPage, error) { + // Try as data_source_id directly + records, err := c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", id)) + if err == nil { + return records, nil + } + + // If 404, id might be a database container ID — resolve to data_source_id + info, dbErr := c.GetDatabaseInfo(ctx, id) + if dbErr != nil { + return nil, fmt.Errorf("query database %s: not a data_source (%v) and not a database (%v)", id, err, dbErr) + } + if info.DataSourceID == "" { + return nil, fmt.Errorf("database %s has no data sources", id) + } + return c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", info.DataSourceID)) +} + +// ResolveBlock re-fetches a single block to resolve file_upload URLs. +// When a block contains a file_upload type, re-fetching it returns the resolved +// download URL (temporary S3 signed URL, 1-hour expiry). +func (c *notionClient) ResolveBlock(ctx context.Context, blockID string) (*notionBlock, error) { + respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/blocks/"+blockID, nil) + if err != nil { + return nil, err + } + var block notionBlock + if err := json.Unmarshal(respBody, &block); err != nil { + return nil, fmt.Errorf("unmarshal block: %w", err) + } + return &block, nil +} + +const maxDownloadSize = 100 * 1024 * 1024 // 100MB — prevent OOM from oversized files + +// DownloadFile downloads a file from the given URL (typically an S3 signed URL). +// Does not go through the rate limiter since it's not a Notion API call. +func (c *notionClient) DownloadFile(ctx context.Context, fileURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil) + if err != nil { + return nil, fmt.Errorf("create download request: %w", err) + } + + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = err + if attempt < maxRetries { + if sErr := sleepWithContext(ctx, time.Duration(1<= 500 && attempt < maxRetries { + if sErr := sleepWithContext(ctx, time.Duration(1< maxDownloadSize { + return nil, fmt.Errorf("file exceeds maximum download size (%d MB)", maxDownloadSize/(1024*1024)) + } + return data, nil + } + + return nil, fmt.Errorf("download file: %w", lastErr) +} + +// --- Shared pagination helper --- + +// paginatePages fetches all pages from a paginated Notion API endpoint. +func (c *notionClient) paginatePages(ctx context.Context, method, path string) ([]notionPage, error) { + var allPages []notionPage + var startCursor string + + for { + body := map[string]interface{}{ + "page_size": 100, + } + if startCursor != "" { + body["start_cursor"] = startCursor + } + + var respBody []byte + var err error + if method == http.MethodPost { + respBody, err = c.doRequest(ctx, method, path, body) + } else { + p := path + if startCursor != "" { + p += "?start_cursor=" + startCursor + "&page_size=100" + } + respBody, err = c.doRequest(ctx, method, p, nil) + } + if err != nil { + return nil, fmt.Errorf("paginate %s: %w", path, err) + } + + var resp paginatedResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("unmarshal paginated response: %w", err) + } + + var pages []notionPage + if err := json.Unmarshal(resp.Results, &pages); err != nil { + return nil, fmt.Errorf("unmarshal page results: %w", err) + } + + for i := range pages { + pages[i].Title = extractTitle(&pages[i]) + } + + allPages = append(allPages, pages...) + + if !resp.HasMore || resp.NextCursor == "" { + break + } + startCursor = resp.NextCursor + } + + return allPages, nil +} + +// --- Title extraction helpers --- + +func extractTitle(page *notionPage) string { + // Try extracting from properties (works for pages) + if page.RawProperties != nil { + var props map[string]json.RawMessage + if err := json.Unmarshal(page.RawProperties, &props); err == nil { + for _, propRaw := range props { + var prop struct { + Type string `json:"type"` + Title []struct { + PlainText string `json:"plain_text"` + } `json:"title"` + } + if err := json.Unmarshal(propRaw, &prop); err != nil { + continue + } + if prop.Type == "title" && len(prop.Title) > 0 { + return joinPlainText(prop.Title) + } + } + } + } + + // Fallback: top-level title array (works for databases) + if page.RawTitle != nil { + var titleSegments []struct { + PlainText string `json:"plain_text"` + } + if err := json.Unmarshal(page.RawTitle, &titleSegments); err == nil && len(titleSegments) > 0 { + return joinPlainText(titleSegments) + } + } + + return "" +} + +func joinPlainText(segments []struct { + PlainText string `json:"plain_text"` +}) string { + var sb strings.Builder + for _, s := range segments { + sb.WriteString(s.PlainText) + } + return sb.String() +} diff --git a/internal/datasource/connector/notion/client_test.go b/internal/datasource/connector/notion/client_test.go new file mode 100644 index 00000000..1eeec22e --- /dev/null +++ b/internal/datasource/connector/notion/client_test.go @@ -0,0 +1,335 @@ +package notion + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// fakeNotion builds an httptest.Server that emulates relevant Notion API endpoints. +func fakeNotion() (*httptest.Server, *Config) { + mux := http.NewServeMux() + + // GET /v1/users/me — auth check + mux.HandleFunc("/v1/users/me", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer test-token" { + w.WriteHeader(401) + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": 401, + "code": "unauthorized", + "message": "API token is invalid.", + }) + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "user", + "id": "bot-user-id", + "type": "bot", + "name": "Test Integration", + }) + }) + + // POST /v1/search — return pages and databases + mux.HandleFunc("/v1/search", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "list", + "results": []interface{}{ + map[string]interface{}{ + "id": "page-1", + "object": "page", + "url": "https://notion.so/Page-1", + "last_edited_time": "2026-01-15T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "workspace", "workspace": true}, + "properties": map[string]interface{}{ + "title": map[string]interface{}{ + "type": "title", + "title": []interface{}{ + map[string]interface{}{"plain_text": "Test Page"}, + }, + }, + }, + }, + map[string]interface{}{ + "id": "db-1", + "object": "data_source", + "url": "https://notion.so/DB-1", + "last_edited_time": "2026-01-16T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "workspace", "workspace": true}, + "title": []interface{}{ + map[string]interface{}{"plain_text": "Test Database"}, + }, + }, + }, + "has_more": false, + "next_cursor": nil, + }) + }) + + // GET /v1/blocks/{id}/children — return blocks + mux.HandleFunc("/v1/blocks/page-1/children", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "list", + "results": []interface{}{ + map[string]interface{}{ + "id": "blk-1", + "type": "paragraph", + "has_children": false, + "paragraph": map[string]interface{}{ + "rich_text": []interface{}{ + map[string]interface{}{ + "type": "text", + "plain_text": "Hello world", + "text": map[string]interface{}{"content": "Hello world"}, + "annotations": map[string]interface{}{ + "bold": false, "italic": false, "strikethrough": false, + "underline": false, "code": false, "color": "default", + }, + }, + }, + }, + }, + map[string]interface{}{ + "id": "blk-2", + "type": "child_page", + "has_children": true, + "child_page": map[string]interface{}{"title": "Sub Page"}, + }, + }, + "has_more": false, + "next_cursor": nil, + }) + }) + + // GET /v1/pages/{id} + mux.HandleFunc("/v1/pages/page-1", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "page-1", + "object": "page", + "url": "https://notion.so/Page-1", + "last_edited_time": "2026-01-15T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "workspace", "workspace": true}, + "properties": map[string]interface{}{ + "title": map[string]interface{}{ + "type": "title", + "title": []interface{}{ + map[string]interface{}{"plain_text": "Test Page"}, + }, + }, + }, + }) + }) + + // GET /v1/databases/{id} — returns container with data_sources array (2025-09-03+) + mux.HandleFunc("/v1/databases/db-1", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "db-1", + "object": "database", + "url": "https://notion.so/DB-1", + "last_edited_time": "2026-01-16T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "workspace", "workspace": true}, + "title": []interface{}{map[string]interface{}{"plain_text": "Test Database"}}, + "data_sources": []interface{}{ + map[string]interface{}{"id": "ds-1", "name": "Default"}, + }, + }) + }) + + // GET /v1/data_sources/{id} — returns schema/properties + mux.HandleFunc("/v1/data_sources/ds-1", func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "ds-1", + "object": "data_source", + "properties": map[string]interface{}{ + "Name": map[string]interface{}{"type": "title", "title": map[string]interface{}{}}, + "Status": map[string]interface{}{"type": "select", "select": map[string]interface{}{}}, + }, + }) + return + } + http.NotFound(w, r) + }) + + // POST /v1/data_sources/{id}/query — return database records (2025-09-03+) + mux.HandleFunc("/v1/data_sources/ds-1/query", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "list", + "results": []interface{}{ + map[string]interface{}{ + "id": "record-1", + "object": "page", + "url": "https://notion.so/Record-1", + "last_edited_time": "2026-01-17T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"}, + "properties": map[string]interface{}{ + "Name": map[string]interface{}{ + "type": "title", + "title": []interface{}{ + map[string]interface{}{"plain_text": "Record One"}, + }, + }, + "Status": map[string]interface{}{ + "type": "select", + "select": map[string]interface{}{"name": "Done"}, + }, + }, + }, + }, + "has_more": false, + "next_cursor": nil, + }) + }) + + // GET /v1/pages/record-1 — direct fetch of a database row (single-select case). + // fetchPage must recognize both database_id (older responses) and + // data_source_id (2025-09-03+) as a record and route to buildRecordItem. + mux.HandleFunc("/v1/pages/record-1", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "id": "record-1", + "object": "page", + "url": "https://notion.so/Record-1", + "last_edited_time": "2026-01-17T10:00:00.000Z", + "in_trash": false, + "parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"}, + "properties": map[string]interface{}{ + "Name": map[string]interface{}{ + "type": "title", + "title": []interface{}{ + map[string]interface{}{"plain_text": "Record One"}, + }, + }, + "Status": map[string]interface{}{ + "type": "select", + "select": map[string]interface{}{"name": "Done"}, + }, + }, + }) + }) + + // GET /v1/blocks/record-1/children — empty blocks for database record + mux.HandleFunc("/v1/blocks/record-1/children", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "list", + "results": []interface{}{}, + "has_more": false, + "next_cursor": nil, + }) + }) + + ts := httptest.NewServer(mux) + cfg := &Config{APIKey: "test-token"} + return ts, cfg +} + +func TestClientPing(t *testing.T) { + ts, cfg := fakeNotion() + defer ts.Close() + + client := newClient(cfg.APIKey, ts.URL) + if err := client.Ping(context.Background()); err != nil { + t.Fatalf("Ping() error: %v", err) + } +} + +func TestClientPing_InvalidToken(t *testing.T) { + ts, _ := fakeNotion() + defer ts.Close() + + client := newClient("wrong-token", ts.URL) + err := client.Ping(context.Background()) + if err == nil { + t.Fatal("expected error for invalid token") + } +} + +func TestClientSearchPages(t *testing.T) { + ts, cfg := fakeNotion() + defer ts.Close() + + client := newClient(cfg.APIKey, ts.URL) + pages, err := client.SearchPages(context.Background()) + if err != nil { + t.Fatalf("SearchPages() error: %v", err) + } + if len(pages) != 2 { + t.Fatalf("expected 2 pages, got %d", len(pages)) + } + // First result is a page + if pages[0].Object != "page" || pages[0].ID != "page-1" { + t.Errorf("page[0] = %+v", pages[0]) + } + // Second result is a data_source (databases are returned as object="data_source" + // in API 2025-09-03+ which is what the mock simulates) + if pages[1].Object != "data_source" || pages[1].ID != "db-1" { + t.Errorf("page[1] = %+v", pages[1]) + } +} + +func TestClientGetBlockChildrenAll(t *testing.T) { + ts, cfg := fakeNotion() + defer ts.Close() + + client := newClient(cfg.APIKey, ts.URL) + blocks, err := client.GetBlockChildrenAll(context.Background(), "page-1") + if err != nil { + t.Fatalf("GetBlockChildrenAll() error: %v", err) + } + if len(blocks) != 2 { + t.Fatalf("expected 2 blocks, got %d", len(blocks)) + } + if blocks[0].Type != "paragraph" { + t.Errorf("block[0].Type = %q", blocks[0].Type) + } + if blocks[1].Type != "child_page" { + t.Errorf("block[1].Type = %q", blocks[1].Type) + } + // child_page should NOT have its children recursed + if blocks[1].Children != nil { + t.Error("child_page children should not be recursed by client") + } +} + +func TestClientGetPage(t *testing.T) { + ts, cfg := fakeNotion() + defer ts.Close() + + client := newClient(cfg.APIKey, ts.URL) + page, err := client.GetPage(context.Background(), "page-1") + if err != nil { + t.Fatalf("GetPage() error: %v", err) + } + if page.ID != "page-1" { + t.Errorf("ID = %q", page.ID) + } + if page.Title != "Test Page" { + t.Errorf("Title = %q, want %q", page.Title, "Test Page") + } +} + +func TestClientQueryDatabaseAll(t *testing.T) { + ts, cfg := fakeNotion() + defer ts.Close() + + client := newClient(cfg.APIKey, ts.URL) + records, err := client.QueryDatabaseAll(context.Background(), "db-1") + if err != nil { + t.Fatalf("QueryDatabaseAll() error: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected 1 record, got %d", len(records)) + } + if records[0].ID != "record-1" { + t.Errorf("record ID = %q", records[0].ID) + } +} + +// Ensure time import is used (referenced by fakeNotion indirectly via time.Time in types). +var _ = time.Now diff --git a/internal/datasource/connector/notion/types.go b/internal/datasource/connector/notion/types.go new file mode 100644 index 00000000..b74d50d3 --- /dev/null +++ b/internal/datasource/connector/notion/types.go @@ -0,0 +1,273 @@ +// Package notion implements the Notion data source connector for WeKnora. +// +// It syncs pages, databases, and attachments from Notion workspaces into WeKnora knowledge bases. +// +// Notion API docs: +// - Authentication: https://developers.notion.com/docs/authorization +// - Search: https://developers.notion.com/reference/post-search +// - Pages: https://developers.notion.com/reference/retrieve-a-page +// - Blocks: https://developers.notion.com/reference/retrieve-a-block +// - Databases: https://developers.notion.com/reference/retrieve-a-database +package notion + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/types" +) + +// NotionAPIVersion is the Notion API version used by this connector. +const NotionAPIVersion = "2026-03-11" + +// DefaultBaseURL is the Notion API base URL. +const DefaultBaseURL = "https://api.notion.com" + +// Config holds Notion-specific configuration for the data source connector. +type Config struct { + APIKey string `json:"api_key"` // Internal Integration Token +} + +// parseNotionConfig extracts and validates Notion config from DataSourceConfig. +func parseNotionConfig(config *types.DataSourceConfig) (*Config, error) { + if config == nil { + return nil, datasource.ErrInvalidConfig + } + + tokenVal, ok := config.Credentials["api_key"] + if !ok { + return nil, fmt.Errorf("%w: missing api_key", datasource.ErrInvalidCredentials) + } + token, ok := tokenVal.(string) + if !ok || token == "" { + return nil, fmt.Errorf("%w: api_key must be a non-empty string", datasource.ErrInvalidCredentials) + } + + return &Config{APIKey: token}, nil +} + +// --- API response types --- + +// notionPage represents a Notion page or database object. +type notionPage struct { + ID string `json:"id"` + Object string `json:"object"` // "page" | "database" | "data_source" (2025-09-03+) + Parent notionParent `json:"parent"` + URL string `json:"url"` + LastEditedTime time.Time `json:"last_edited_time"` + InTrash bool `json:"in_trash"` + // Title is extracted from properties after unmarshaling; not directly from JSON. + Title string `json:"-"` + // RawTitle holds the top-level title array (used by database objects). + RawTitle json.RawMessage `json:"title,omitempty"` + // RawProperties holds the raw JSON for property extraction. + RawProperties json.RawMessage `json:"properties"` + // DatabaseParent indicates where the database lives in the workspace hierarchy. + // Only present on data_source objects (API 2025-09-03+). For example, if a database + // is inside a page, this will be {type: "page_id", page_id: "..."}. + DatabaseParent *notionParent `json:"database_parent,omitempty"` +} + +// Parent type constants for notionParent.Type +const ( + parentTypeWorkspace = "workspace" + parentTypePageID = "page_id" + parentTypeDatabaseID = "database_id" + parentTypeDataSourceID = "data_source_id" + parentTypeBlockID = "block_id" +) + +// notionParent represents the parent relationship of a page/database. +type notionParent struct { + Type string `json:"type"` + PageID string `json:"page_id,omitempty"` + DatabaseID string `json:"database_id,omitempty"` + DataSourceID string `json:"data_source_id,omitempty"` + BlockID string `json:"block_id,omitempty"` +} + +// GetParentID returns the effective parent ID regardless of parent type. +func (p *notionParent) GetParentID() string { + switch p.Type { + case parentTypePageID: + return p.PageID + case parentTypeDatabaseID: + return p.DatabaseID + case parentTypeDataSourceID: + return p.DataSourceID + case parentTypeBlockID: + return p.BlockID + default: + return "" + } +} + +// isDatabase returns true if the notionPage represents a database or data source. +func (p *notionPage) isDatabase() bool { + return p.Object == "database" || p.Object == "data_source" +} + +// notionBlock represents a single content block in a Notion page. +// Requires custom UnmarshalJSON to extract the type-specific content. +type notionBlock struct { + ID string `json:"id"` + Type string `json:"type"` + HasChildren bool `json:"has_children"` + RawContent json.RawMessage `json:"-"` // Extracted from the type-named field (e.g., "paragraph") + Children []notionBlock `json:"-"` // Populated by client.GetBlockChildrenAll, not from API directly +} + +// UnmarshalJSON implements custom JSON unmarshaling for notionBlock. +// It extracts the dynamic type-keyed field (e.g., {"type":"paragraph","paragraph":{...}}) +// into RawContent. +func (b *notionBlock) UnmarshalJSON(data []byte) error { + // Use a helper struct to avoid infinite recursion + type blockAlias struct { + ID string `json:"id"` + Type string `json:"type"` + HasChildren bool `json:"has_children"` + } + + var alias blockAlias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + b.ID = alias.ID + b.Type = alias.Type + b.HasChildren = alias.HasChildren + + // Extract the type-named field into RawContent + if alias.Type != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + if content, ok := raw[alias.Type]; ok { + b.RawContent = content + } + } + + return nil +} + +// notionRichText represents a rich text element in Notion. +type notionRichText struct { + Type string `json:"type"` // "text" | "mention" | "equation" + PlainText string `json:"plain_text"` + Href string `json:"href,omitempty"` + Annotations notionAnnotations `json:"annotations"` + Text *notionTextContent `json:"text,omitempty"` + Mention *notionMention `json:"mention,omitempty"` + Equation *notionEquation `json:"equation,omitempty"` +} + +// notionAnnotations holds text style information. +type notionAnnotations struct { + Bold bool `json:"bold"` + Italic bool `json:"italic"` + Strikethrough bool `json:"strikethrough"` + Underline bool `json:"underline"` + Code bool `json:"code"` + Color string `json:"color"` +} + +// notionTextContent holds the content for "text" type rich text. +type notionTextContent struct { + Content string `json:"content"` + Link *struct { + URL string `json:"url"` + } `json:"link,omitempty"` +} + +// notionMention holds the content for "mention" type rich text. +type notionMention struct { + Type string `json:"type"` // "page" | "database" | "date" | "user" | "link_preview" + Page *struct { + ID string `json:"id"` + } `json:"page,omitempty"` + Database *struct { + ID string `json:"id"` + } `json:"database,omitempty"` + Date *notionDateMention `json:"date,omitempty"` + LinkPreview *struct { + URL string `json:"url"` + } `json:"link_preview,omitempty"` +} + +// notionDateMention holds date information for a mention. +type notionDateMention struct { + Start string `json:"start"` + End string `json:"end,omitempty"` +} + +// notionEquation holds the content for "equation" type rich text. +type notionEquation struct { + Expression string `json:"expression"` +} + +// notionFile represents a file object in Notion (used by image, file, pdf blocks). +type notionFile struct { + Type string `json:"type"` // "file" | "external" | "file_upload" + File *struct { + URL string `json:"url"` + ExpiryTime time.Time `json:"expiry_time"` + } `json:"file,omitempty"` + External *struct { + URL string `json:"url"` + } `json:"external,omitempty"` + FileUpload *struct { + ID string `json:"id"` + } `json:"file_upload,omitempty"` + Caption []notionRichText `json:"caption,omitempty"` + Name string `json:"name,omitempty"` +} + +// GetURL returns the file URL regardless of type (hosted or external). +// For file_upload type, returns empty — caller must resolve via client.GetFileUploadURL(). +func (f *notionFile) GetURL() string { + if f.File != nil { + return f.File.URL + } + if f.External != nil { + return f.External.URL + } + return "" +} + +// GetFileUploadID returns the file upload ID for file_upload type, empty otherwise. +func (f *notionFile) GetFileUploadID() string { + if f.FileUpload != nil { + return f.FileUpload.ID + } + return "" +} + +// --- Cursor --- + +// notionCursor tracks state for incremental sync. +// Only PageEditTimes is used for diffing; LastSyncTime lives on types.SyncCursor. +type notionCursor struct { + PageEditTimes map[string]time.Time `json:"page_edit_times"` // page_id → last_edited_time +} + +// --- Attachment --- + +// attachment represents a file to be downloaded from a Notion page. +type attachment struct { + URL string // Notion S3 signed URL (expires in 1 hour) + FileName string + Type string // "image" | "file" | "pdf" | "video" | "audio" +} + +// --- Pagination --- + +// paginatedResponse is the common response wrapper for paginated Notion API responses. +type paginatedResponse struct { + Object string `json:"object"` // "list" + Results json.RawMessage `json:"results"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor,omitempty"` +} + diff --git a/internal/datasource/connector/notion/types_test.go b/internal/datasource/connector/notion/types_test.go new file mode 100644 index 00000000..b61018b4 --- /dev/null +++ b/internal/datasource/connector/notion/types_test.go @@ -0,0 +1,144 @@ +package notion + +import ( + "encoding/json" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestParseNotionConfig(t *testing.T) { + t.Run("valid", func(t *testing.T) { + cfg, err := parseNotionConfig(&types.DataSourceConfig{ + Credentials: map[string]interface{}{ + "api_key": "ntn_test123", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.APIKey != "ntn_test123" { + t.Errorf("APIKey = %q, want %q", cfg.APIKey, "ntn_test123") + } + }) + + t.Run("nil config", func(t *testing.T) { + _, err := parseNotionConfig(nil) + if err == nil { + t.Fatal("expected error for nil config") + } + }) + + t.Run("missing api_key", func(t *testing.T) { + _, err := parseNotionConfig(&types.DataSourceConfig{ + Credentials: map[string]interface{}{}, + }) + if err == nil { + t.Fatal("expected error for missing api_key") + } + }) +} + +func TestNotionBlockUnmarshalJSON(t *testing.T) { + t.Run("paragraph block", func(t *testing.T) { + data := []byte(`{ + "id": "block-1", + "type": "paragraph", + "has_children": false, + "paragraph": {"rich_text": [{"type": "text", "plain_text": "hello"}]} + }`) + var block notionBlock + if err := json.Unmarshal(data, &block); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if block.ID != "block-1" { + t.Errorf("ID = %q, want %q", block.ID, "block-1") + } + if block.Type != "paragraph" { + t.Errorf("Type = %q, want %q", block.Type, "paragraph") + } + if block.HasChildren != false { + t.Error("HasChildren should be false") + } + if block.RawContent == nil { + t.Fatal("RawContent should not be nil") + } + // Verify RawContent contains the paragraph object + var content struct { + RichText []notionRichText `json:"rich_text"` + } + if err := json.Unmarshal(block.RawContent, &content); err != nil { + t.Fatalf("unmarshal RawContent: %v", err) + } + if len(content.RichText) != 1 || content.RichText[0].PlainText != "hello" { + t.Errorf("unexpected rich_text content: %+v", content.RichText) + } + }) + + t.Run("block with children flag", func(t *testing.T) { + data := []byte(`{ + "id": "block-2", + "type": "toggle", + "has_children": true, + "toggle": {"rich_text": [{"type": "text", "plain_text": "Details"}]} + }`) + var block notionBlock + if err := json.Unmarshal(data, &block); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if !block.HasChildren { + t.Error("HasChildren should be true") + } + // Children is populated by client, not by unmarshal + if block.Children != nil { + t.Error("Children should be nil after unmarshal") + } + }) + + t.Run("unknown block type", func(t *testing.T) { + data := []byte(`{"id": "block-3", "type": "future_type", "has_children": false}`) + var block notionBlock + if err := json.Unmarshal(data, &block); err != nil { + t.Fatalf("unmarshal error: %v", err) + } + if block.Type != "future_type" { + t.Errorf("Type = %q", block.Type) + } + // RawContent may be nil for unknown types without matching field + }) +} + +func TestNotionFileGetURL(t *testing.T) { + t.Run("hosted file", func(t *testing.T) { + f := ¬ionFile{ + Type: "file", + File: &struct { + URL string `json:"url"` + ExpiryTime time.Time `json:"expiry_time"` + }{URL: "https://s3.example.com/file.pdf"}, + } + if got := f.GetURL(); got != "https://s3.example.com/file.pdf" { + t.Errorf("GetURL() = %q", got) + } + }) + + t.Run("external file", func(t *testing.T) { + f := ¬ionFile{ + Type: "external", + External: &struct { + URL string `json:"url"` + }{URL: "https://example.com/image.png"}, + } + if got := f.GetURL(); got != "https://example.com/image.png" { + t.Errorf("GetURL() = %q", got) + } + }) + + t.Run("empty file", func(t *testing.T) { + f := ¬ionFile{Type: "file"} + if got := f.GetURL(); got != "" { + t.Errorf("GetURL() = %q, want empty", got) + } + }) +} diff --git a/internal/types/datasource.go b/internal/types/datasource.go index 655e506e..b4f7604b 100644 --- a/internal/types/datasource.go +++ b/internal/types/datasource.go @@ -232,6 +232,9 @@ type Resource struct { // For hierarchical resources (parent ID if applicable) ParentID string `json:"parent_id,omitempty"` + // Whether this resource has children that can be expanded + HasChildren bool `json:"has_children,omitempty"` + // Additional metadata Metadata map[string]interface{} `json:"metadata,omitempty"` } diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index b78ede1c..6e5a8f52 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -28,6 +28,7 @@ const ( ChannelDingtalk = "dingtalk" // DingTalk ChannelSlack = "slack" // Slack ChannelIM = "im" // Generic IM channel + ChannelNotion = "notion" // Notion ) // Knowledge parse status constants