feat(notion): add Notion API client and type definitions

- Define notionPage, notionBlock, notionParent, notionFile and cursor types
- Implement rate-limited HTTP client with context-aware retry and backoff
- Support search, page/block retrieval, database queries, and file downloads
- Add HasChildren field to Resource type, NotionChannel to knowledge types
- Add unit tests for types and client
This commit is contained in:
nullkey
2026-04-06 23:31:38 +08:00
committed by lyingbug
parent d5ecc150e0
commit 9f35abafff
6 changed files with 1279 additions and 0 deletions
@@ -0,0 +1,523 @@
package notion
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
)
// notionClient wraps the Notion API with rate limiting and retry logic.
type notionClient struct {
token string
httpClient *http.Client
limiter *rate.Limiter
baseURL string
}
// newClient creates a new Notion API client.
func newClient(token, baseURL string) *notionClient {
if baseURL == "" {
baseURL = DefaultBaseURL
}
return &notionClient{
token: token,
httpClient: &http.Client{Timeout: 30 * time.Second},
limiter: rate.NewLimiter(rate.Limit(3), 3), // 3 req/s with burst of 3 (matches Notion's actual behavior)
baseURL: baseURL,
}
}
const maxRetries = 3
// sleepWithContext pauses for the given duration, returning early if ctx is cancelled.
func sleepWithContext(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// doRequest performs an authenticated, rate-limited HTTP request to the Notion API.
func (c *notionClient) doRequest(ctx context.Context, method, path string, body interface{}) ([]byte, error) {
if err := c.limiter.Wait(ctx); err != nil {
return nil, fmt.Errorf("rate limiter: %w", err)
}
var bodyReader io.Reader
if body != nil {
bodyBytes, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(bodyBytes)
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bodyReader)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.token)
req.Header.Set("Notion-Version", NotionAPIVersion)
req.Header.Set("Content-Type", "application/json")
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
if attempt > 0 {
if body != nil {
bodyBytes, _ := json.Marshal(body)
req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
}
}
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = err
if attempt < maxRetries {
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
return nil, sErr
}
continue
}
break
}
respBody, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return respBody, nil
case resp.StatusCode == 401 || resp.StatusCode == 403:
return nil, fmt.Errorf("%w: %s", datasource.ErrInvalidCredentials, string(respBody))
case resp.StatusCode == 404:
return nil, fmt.Errorf("%w: %s", datasource.ErrResourceNotFound, path)
case resp.StatusCode == 429:
retryAfter := resp.Header.Get("Retry-After")
wait := 1 * time.Second
if secs, err := strconv.ParseFloat(retryAfter, 64); err == nil && secs > 0 {
wait = time.Duration(secs * float64(time.Second))
}
logger.Warnf(ctx, "[Notion] rate limited, retry after %v (attempt %d/%d)", wait, attempt+1, maxRetries)
lastErr = fmt.Errorf("rate limited: %s", string(respBody))
if attempt < maxRetries {
if sErr := sleepWithContext(ctx, wait); sErr != nil {
return nil, sErr
}
continue
}
case resp.StatusCode >= 500:
lastErr = fmt.Errorf("server error %d: %s", resp.StatusCode, string(respBody))
if attempt < maxRetries {
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
return nil, sErr
}
continue
}
default:
return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody))
}
}
if lastErr != nil {
return nil, fmt.Errorf("%w: %v", datasource.ErrFetchFailed, lastErr)
}
return nil, datasource.ErrFetchFailed
}
// Ping verifies the API token is valid by calling GET /v1/users/me.
func (c *notionClient) Ping(ctx context.Context) error {
_, err := c.doRequest(ctx, http.MethodGet, "/v1/users/me", nil)
return err
}
// SearchPages returns all pages and databases accessible to the integration.
func (c *notionClient) SearchPages(ctx context.Context) ([]notionPage, error) {
return c.paginatePages(ctx, http.MethodPost, "/v1/search")
}
// GetPage retrieves a single page by ID.
func (c *notionClient) GetPage(ctx context.Context, pageID string) (*notionPage, error) {
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/pages/"+pageID, nil)
if err != nil {
return nil, err
}
var page notionPage
if err := json.Unmarshal(respBody, &page); err != nil {
return nil, fmt.Errorf("unmarshal page: %w", err)
}
page.Title = extractTitle(&page)
return &page, nil
}
// databaseInfo holds both the metadata and the data source ID from a single API call.
type databaseInfo struct {
Page notionPage
DataSourceID string
}
// GetDatabaseInfo retrieves a database container by ID, returning both
// the page metadata and the primary data source ID in a single API call.
func (c *notionClient) GetDatabaseInfo(ctx context.Context, dbID string) (*databaseInfo, error) {
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/databases/"+dbID, nil)
if err != nil {
return nil, err
}
var db notionPage
if err := json.Unmarshal(respBody, &db); err != nil {
return nil, fmt.Errorf("unmarshal database: %w", err)
}
db.Title = extractTitle(&db)
var dsResult struct {
DataSources []struct {
ID string `json:"id"`
} `json:"data_sources"`
}
if err := json.Unmarshal(respBody, &dsResult); err != nil {
return nil, fmt.Errorf("unmarshal data_sources: %w", err)
}
dsID := ""
if len(dsResult.DataSources) > 0 {
dsID = dsResult.DataSources[0].ID
}
return &databaseInfo{Page: db, DataSourceID: dsID}, nil
}
// GetDataSourceInfo retrieves a data source by ID, returning its metadata.
// In API 2025-09-03+, data_source objects hold the schema/properties and are
// the target for record queries. The response includes database_parent to
// locate the database in the workspace hierarchy.
func (c *notionClient) GetDataSourceInfo(ctx context.Context, dsID string) (*notionPage, error) {
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/data_sources/"+dsID, nil)
if err != nil {
return nil, err
}
var ds notionPage
if err := json.Unmarshal(respBody, &ds); err != nil {
return nil, fmt.Errorf("unmarshal data_source: %w", err)
}
ds.Title = extractTitle(&ds)
return &ds, nil
}
// GetBlockChildrenFlat fetches only the direct children of a block (no recursion).
// Used by discoverPages to quickly scan for child_page/child_database without
// fetching the full block tree content.
func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string) ([]notionBlock, error) {
var allBlocks []notionBlock
var startCursor string
for {
path := fmt.Sprintf("/v1/blocks/%s/children", blockID)
if startCursor != "" {
path += "?start_cursor=" + startCursor
}
respBody, err := c.doRequest(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, fmt.Errorf("get block children for %s: %w", blockID, err)
}
var resp paginatedResponse
if err := json.Unmarshal(respBody, &resp); err != nil {
return nil, fmt.Errorf("unmarshal block children response: %w", err)
}
var blocks []notionBlock
if err := json.Unmarshal(resp.Results, &blocks); err != nil {
return nil, fmt.Errorf("unmarshal blocks: %w", err)
}
allBlocks = append(allBlocks, blocks...)
if !resp.HasMore || resp.NextCursor == "" {
break
}
startCursor = resp.NextCursor
}
return allBlocks, nil
}
const maxBlockDepth = 5 // Limit recursion depth — deeper content has diminishing value for knowledge bases
const maxBlocksPerPage = 1000 // Limit total blocks fetched per page to prevent runaway API calls
// GetBlockChildrenAll recursively fetches all blocks under a given block ID,
// building a tree structure with Children populated for blocks with has_children=true.
// child_page and child_database blocks are NOT recursed into (handled by connector layer).
// Recursion is limited to maxBlockDepth to prevent excessive API calls on complex pages.
func (c *notionClient) GetBlockChildrenAll(ctx context.Context, blockID string) ([]notionBlock, error) {
return c.getBlockChildrenRecursive(ctx, blockID, 0)
}
func (c *notionClient) getBlockChildrenRecursive(ctx context.Context, blockID string, depth int) ([]notionBlock, error) {
var allBlocks []notionBlock
var startCursor string
for {
path := fmt.Sprintf("/v1/blocks/%s/children", blockID)
if startCursor != "" {
path += "?start_cursor=" + startCursor
}
respBody, err := c.doRequest(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, fmt.Errorf("get block children for %s: %w", blockID, err)
}
var resp paginatedResponse
if err := json.Unmarshal(respBody, &resp); err != nil {
return nil, fmt.Errorf("unmarshal block children response: %w", err)
}
var blocks []notionBlock
if err := json.Unmarshal(resp.Results, &blocks); err != nil {
return nil, fmt.Errorf("unmarshal blocks: %w", err)
}
allBlocks = append(allBlocks, blocks...)
if len(allBlocks) >= maxBlocksPerPage || !resp.HasMore || resp.NextCursor == "" {
break
}
startCursor = resp.NextCursor
}
if depth >= maxBlockDepth {
return allBlocks, nil
}
for i := range allBlocks {
if !allBlocks[i].HasChildren {
continue
}
// Skip block types that don't contribute useful content
switch allBlocks[i].Type {
case "child_page", "child_database", "unsupported", "template", "breadcrumb", "table_of_contents":
continue
}
children, err := c.getBlockChildrenRecursive(ctx, allBlocks[i].ID, depth+1)
if err != nil {
logger.Warnf(ctx, "[Notion] failed to get children for block %s (depth %d): %v", allBlocks[i].ID, depth, err)
continue
}
allBlocks[i].Children = children
}
return allBlocks, nil
}
// QueryDatabaseAll retrieves all records from a database via POST /v1/data_sources/{id}/query.
// Accepts either a data_source_id (from search) or a database_id (from child_database blocks).
// For database_ids, resolves to data_source_id via GET /v1/databases/{id}.
func (c *notionClient) QueryDatabaseAll(ctx context.Context, id string) ([]notionPage, error) {
// Try as data_source_id directly
records, err := c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", id))
if err == nil {
return records, nil
}
// If 404, id might be a database container ID — resolve to data_source_id
info, dbErr := c.GetDatabaseInfo(ctx, id)
if dbErr != nil {
return nil, fmt.Errorf("query database %s: not a data_source (%v) and not a database (%v)", id, err, dbErr)
}
if info.DataSourceID == "" {
return nil, fmt.Errorf("database %s has no data sources", id)
}
return c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", info.DataSourceID))
}
// ResolveBlock re-fetches a single block to resolve file_upload URLs.
// When a block contains a file_upload type, re-fetching it returns the resolved
// download URL (temporary S3 signed URL, 1-hour expiry).
func (c *notionClient) ResolveBlock(ctx context.Context, blockID string) (*notionBlock, error) {
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/blocks/"+blockID, nil)
if err != nil {
return nil, err
}
var block notionBlock
if err := json.Unmarshal(respBody, &block); err != nil {
return nil, fmt.Errorf("unmarshal block: %w", err)
}
return &block, nil
}
const maxDownloadSize = 100 * 1024 * 1024 // 100MB — prevent OOM from oversized files
// DownloadFile downloads a file from the given URL (typically an S3 signed URL).
// Does not go through the rate limiter since it's not a Notion API call.
func (c *notionClient) DownloadFile(ctx context.Context, fileURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
if err != nil {
return nil, fmt.Errorf("create download request: %w", err)
}
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = err
if attempt < maxRetries {
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
return nil, sErr
}
continue
}
break
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
lastErr = fmt.Errorf("download failed with status %d", resp.StatusCode)
if resp.StatusCode >= 500 && attempt < maxRetries {
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
return nil, sErr
}
continue
}
break
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxDownloadSize+1))
resp.Body.Close()
if err != nil {
return nil, err
}
if int64(len(data)) > maxDownloadSize {
return nil, fmt.Errorf("file exceeds maximum download size (%d MB)", maxDownloadSize/(1024*1024))
}
return data, nil
}
return nil, fmt.Errorf("download file: %w", lastErr)
}
// --- Shared pagination helper ---
// paginatePages fetches all pages from a paginated Notion API endpoint.
func (c *notionClient) paginatePages(ctx context.Context, method, path string) ([]notionPage, error) {
var allPages []notionPage
var startCursor string
for {
body := map[string]interface{}{
"page_size": 100,
}
if startCursor != "" {
body["start_cursor"] = startCursor
}
var respBody []byte
var err error
if method == http.MethodPost {
respBody, err = c.doRequest(ctx, method, path, body)
} else {
p := path
if startCursor != "" {
p += "?start_cursor=" + startCursor + "&page_size=100"
}
respBody, err = c.doRequest(ctx, method, p, nil)
}
if err != nil {
return nil, fmt.Errorf("paginate %s: %w", path, err)
}
var resp paginatedResponse
if err := json.Unmarshal(respBody, &resp); err != nil {
return nil, fmt.Errorf("unmarshal paginated response: %w", err)
}
var pages []notionPage
if err := json.Unmarshal(resp.Results, &pages); err != nil {
return nil, fmt.Errorf("unmarshal page results: %w", err)
}
for i := range pages {
pages[i].Title = extractTitle(&pages[i])
}
allPages = append(allPages, pages...)
if !resp.HasMore || resp.NextCursor == "" {
break
}
startCursor = resp.NextCursor
}
return allPages, nil
}
// --- Title extraction helpers ---
func extractTitle(page *notionPage) string {
// Try extracting from properties (works for pages)
if page.RawProperties != nil {
var props map[string]json.RawMessage
if err := json.Unmarshal(page.RawProperties, &props); err == nil {
for _, propRaw := range props {
var prop struct {
Type string `json:"type"`
Title []struct {
PlainText string `json:"plain_text"`
} `json:"title"`
}
if err := json.Unmarshal(propRaw, &prop); err != nil {
continue
}
if prop.Type == "title" && len(prop.Title) > 0 {
return joinPlainText(prop.Title)
}
}
}
}
// Fallback: top-level title array (works for databases)
if page.RawTitle != nil {
var titleSegments []struct {
PlainText string `json:"plain_text"`
}
if err := json.Unmarshal(page.RawTitle, &titleSegments); err == nil && len(titleSegments) > 0 {
return joinPlainText(titleSegments)
}
}
return ""
}
func joinPlainText(segments []struct {
PlainText string `json:"plain_text"`
}) string {
var sb strings.Builder
for _, s := range segments {
sb.WriteString(s.PlainText)
}
return sb.String()
}
@@ -0,0 +1,335 @@
package notion
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
// fakeNotion builds an httptest.Server that emulates relevant Notion API endpoints.
func fakeNotion() (*httptest.Server, *Config) {
mux := http.NewServeMux()
// GET /v1/users/me — auth check
mux.HandleFunc("/v1/users/me", func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer test-token" {
w.WriteHeader(401)
json.NewEncoder(w).Encode(map[string]interface{}{
"status": 401,
"code": "unauthorized",
"message": "API token is invalid.",
})
return
}
json.NewEncoder(w).Encode(map[string]interface{}{
"object": "user",
"id": "bot-user-id",
"type": "bot",
"name": "Test Integration",
})
})
// POST /v1/search — return pages and databases
mux.HandleFunc("/v1/search", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"object": "list",
"results": []interface{}{
map[string]interface{}{
"id": "page-1",
"object": "page",
"url": "https://notion.so/Page-1",
"last_edited_time": "2026-01-15T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
"properties": map[string]interface{}{
"title": map[string]interface{}{
"type": "title",
"title": []interface{}{
map[string]interface{}{"plain_text": "Test Page"},
},
},
},
},
map[string]interface{}{
"id": "db-1",
"object": "data_source",
"url": "https://notion.so/DB-1",
"last_edited_time": "2026-01-16T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
"title": []interface{}{
map[string]interface{}{"plain_text": "Test Database"},
},
},
},
"has_more": false,
"next_cursor": nil,
})
})
// GET /v1/blocks/{id}/children — return blocks
mux.HandleFunc("/v1/blocks/page-1/children", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"object": "list",
"results": []interface{}{
map[string]interface{}{
"id": "blk-1",
"type": "paragraph",
"has_children": false,
"paragraph": map[string]interface{}{
"rich_text": []interface{}{
map[string]interface{}{
"type": "text",
"plain_text": "Hello world",
"text": map[string]interface{}{"content": "Hello world"},
"annotations": map[string]interface{}{
"bold": false, "italic": false, "strikethrough": false,
"underline": false, "code": false, "color": "default",
},
},
},
},
},
map[string]interface{}{
"id": "blk-2",
"type": "child_page",
"has_children": true,
"child_page": map[string]interface{}{"title": "Sub Page"},
},
},
"has_more": false,
"next_cursor": nil,
})
})
// GET /v1/pages/{id}
mux.HandleFunc("/v1/pages/page-1", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"id": "page-1",
"object": "page",
"url": "https://notion.so/Page-1",
"last_edited_time": "2026-01-15T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
"properties": map[string]interface{}{
"title": map[string]interface{}{
"type": "title",
"title": []interface{}{
map[string]interface{}{"plain_text": "Test Page"},
},
},
},
})
})
// GET /v1/databases/{id} — returns container with data_sources array (2025-09-03+)
mux.HandleFunc("/v1/databases/db-1", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"id": "db-1",
"object": "database",
"url": "https://notion.so/DB-1",
"last_edited_time": "2026-01-16T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
"title": []interface{}{map[string]interface{}{"plain_text": "Test Database"}},
"data_sources": []interface{}{
map[string]interface{}{"id": "ds-1", "name": "Default"},
},
})
})
// GET /v1/data_sources/{id} — returns schema/properties
mux.HandleFunc("/v1/data_sources/ds-1", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(map[string]interface{}{
"id": "ds-1",
"object": "data_source",
"properties": map[string]interface{}{
"Name": map[string]interface{}{"type": "title", "title": map[string]interface{}{}},
"Status": map[string]interface{}{"type": "select", "select": map[string]interface{}{}},
},
})
return
}
http.NotFound(w, r)
})
// POST /v1/data_sources/{id}/query — return database records (2025-09-03+)
mux.HandleFunc("/v1/data_sources/ds-1/query", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"object": "list",
"results": []interface{}{
map[string]interface{}{
"id": "record-1",
"object": "page",
"url": "https://notion.so/Record-1",
"last_edited_time": "2026-01-17T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"},
"properties": map[string]interface{}{
"Name": map[string]interface{}{
"type": "title",
"title": []interface{}{
map[string]interface{}{"plain_text": "Record One"},
},
},
"Status": map[string]interface{}{
"type": "select",
"select": map[string]interface{}{"name": "Done"},
},
},
},
},
"has_more": false,
"next_cursor": nil,
})
})
// GET /v1/pages/record-1 — direct fetch of a database row (single-select case).
// fetchPage must recognize both database_id (older responses) and
// data_source_id (2025-09-03+) as a record and route to buildRecordItem.
mux.HandleFunc("/v1/pages/record-1", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"id": "record-1",
"object": "page",
"url": "https://notion.so/Record-1",
"last_edited_time": "2026-01-17T10:00:00.000Z",
"in_trash": false,
"parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"},
"properties": map[string]interface{}{
"Name": map[string]interface{}{
"type": "title",
"title": []interface{}{
map[string]interface{}{"plain_text": "Record One"},
},
},
"Status": map[string]interface{}{
"type": "select",
"select": map[string]interface{}{"name": "Done"},
},
},
})
})
// GET /v1/blocks/record-1/children — empty blocks for database record
mux.HandleFunc("/v1/blocks/record-1/children", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"object": "list",
"results": []interface{}{},
"has_more": false,
"next_cursor": nil,
})
})
ts := httptest.NewServer(mux)
cfg := &Config{APIKey: "test-token"}
return ts, cfg
}
func TestClientPing(t *testing.T) {
ts, cfg := fakeNotion()
defer ts.Close()
client := newClient(cfg.APIKey, ts.URL)
if err := client.Ping(context.Background()); err != nil {
t.Fatalf("Ping() error: %v", err)
}
}
func TestClientPing_InvalidToken(t *testing.T) {
ts, _ := fakeNotion()
defer ts.Close()
client := newClient("wrong-token", ts.URL)
err := client.Ping(context.Background())
if err == nil {
t.Fatal("expected error for invalid token")
}
}
func TestClientSearchPages(t *testing.T) {
ts, cfg := fakeNotion()
defer ts.Close()
client := newClient(cfg.APIKey, ts.URL)
pages, err := client.SearchPages(context.Background())
if err != nil {
t.Fatalf("SearchPages() error: %v", err)
}
if len(pages) != 2 {
t.Fatalf("expected 2 pages, got %d", len(pages))
}
// First result is a page
if pages[0].Object != "page" || pages[0].ID != "page-1" {
t.Errorf("page[0] = %+v", pages[0])
}
// Second result is a data_source (databases are returned as object="data_source"
// in API 2025-09-03+ which is what the mock simulates)
if pages[1].Object != "data_source" || pages[1].ID != "db-1" {
t.Errorf("page[1] = %+v", pages[1])
}
}
func TestClientGetBlockChildrenAll(t *testing.T) {
ts, cfg := fakeNotion()
defer ts.Close()
client := newClient(cfg.APIKey, ts.URL)
blocks, err := client.GetBlockChildrenAll(context.Background(), "page-1")
if err != nil {
t.Fatalf("GetBlockChildrenAll() error: %v", err)
}
if len(blocks) != 2 {
t.Fatalf("expected 2 blocks, got %d", len(blocks))
}
if blocks[0].Type != "paragraph" {
t.Errorf("block[0].Type = %q", blocks[0].Type)
}
if blocks[1].Type != "child_page" {
t.Errorf("block[1].Type = %q", blocks[1].Type)
}
// child_page should NOT have its children recursed
if blocks[1].Children != nil {
t.Error("child_page children should not be recursed by client")
}
}
func TestClientGetPage(t *testing.T) {
ts, cfg := fakeNotion()
defer ts.Close()
client := newClient(cfg.APIKey, ts.URL)
page, err := client.GetPage(context.Background(), "page-1")
if err != nil {
t.Fatalf("GetPage() error: %v", err)
}
if page.ID != "page-1" {
t.Errorf("ID = %q", page.ID)
}
if page.Title != "Test Page" {
t.Errorf("Title = %q, want %q", page.Title, "Test Page")
}
}
func TestClientQueryDatabaseAll(t *testing.T) {
ts, cfg := fakeNotion()
defer ts.Close()
client := newClient(cfg.APIKey, ts.URL)
records, err := client.QueryDatabaseAll(context.Background(), "db-1")
if err != nil {
t.Fatalf("QueryDatabaseAll() error: %v", err)
}
if len(records) != 1 {
t.Fatalf("expected 1 record, got %d", len(records))
}
if records[0].ID != "record-1" {
t.Errorf("record ID = %q", records[0].ID)
}
}
// Ensure time import is used (referenced by fakeNotion indirectly via time.Time in types).
var _ = time.Now
@@ -0,0 +1,273 @@
// Package notion implements the Notion data source connector for WeKnora.
//
// It syncs pages, databases, and attachments from Notion workspaces into WeKnora knowledge bases.
//
// Notion API docs:
// - Authentication: https://developers.notion.com/docs/authorization
// - Search: https://developers.notion.com/reference/post-search
// - Pages: https://developers.notion.com/reference/retrieve-a-page
// - Blocks: https://developers.notion.com/reference/retrieve-a-block
// - Databases: https://developers.notion.com/reference/retrieve-a-database
package notion
import (
"encoding/json"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
)
// NotionAPIVersion is the Notion API version used by this connector.
const NotionAPIVersion = "2026-03-11"
// DefaultBaseURL is the Notion API base URL.
const DefaultBaseURL = "https://api.notion.com"
// Config holds Notion-specific configuration for the data source connector.
type Config struct {
APIKey string `json:"api_key"` // Internal Integration Token
}
// parseNotionConfig extracts and validates Notion config from DataSourceConfig.
func parseNotionConfig(config *types.DataSourceConfig) (*Config, error) {
if config == nil {
return nil, datasource.ErrInvalidConfig
}
tokenVal, ok := config.Credentials["api_key"]
if !ok {
return nil, fmt.Errorf("%w: missing api_key", datasource.ErrInvalidCredentials)
}
token, ok := tokenVal.(string)
if !ok || token == "" {
return nil, fmt.Errorf("%w: api_key must be a non-empty string", datasource.ErrInvalidCredentials)
}
return &Config{APIKey: token}, nil
}
// --- API response types ---
// notionPage represents a Notion page or database object.
type notionPage struct {
ID string `json:"id"`
Object string `json:"object"` // "page" | "database" | "data_source" (2025-09-03+)
Parent notionParent `json:"parent"`
URL string `json:"url"`
LastEditedTime time.Time `json:"last_edited_time"`
InTrash bool `json:"in_trash"`
// Title is extracted from properties after unmarshaling; not directly from JSON.
Title string `json:"-"`
// RawTitle holds the top-level title array (used by database objects).
RawTitle json.RawMessage `json:"title,omitempty"`
// RawProperties holds the raw JSON for property extraction.
RawProperties json.RawMessage `json:"properties"`
// DatabaseParent indicates where the database lives in the workspace hierarchy.
// Only present on data_source objects (API 2025-09-03+). For example, if a database
// is inside a page, this will be {type: "page_id", page_id: "..."}.
DatabaseParent *notionParent `json:"database_parent,omitempty"`
}
// Parent type constants for notionParent.Type
const (
parentTypeWorkspace = "workspace"
parentTypePageID = "page_id"
parentTypeDatabaseID = "database_id"
parentTypeDataSourceID = "data_source_id"
parentTypeBlockID = "block_id"
)
// notionParent represents the parent relationship of a page/database.
type notionParent struct {
Type string `json:"type"`
PageID string `json:"page_id,omitempty"`
DatabaseID string `json:"database_id,omitempty"`
DataSourceID string `json:"data_source_id,omitempty"`
BlockID string `json:"block_id,omitempty"`
}
// GetParentID returns the effective parent ID regardless of parent type.
func (p *notionParent) GetParentID() string {
switch p.Type {
case parentTypePageID:
return p.PageID
case parentTypeDatabaseID:
return p.DatabaseID
case parentTypeDataSourceID:
return p.DataSourceID
case parentTypeBlockID:
return p.BlockID
default:
return ""
}
}
// isDatabase returns true if the notionPage represents a database or data source.
func (p *notionPage) isDatabase() bool {
return p.Object == "database" || p.Object == "data_source"
}
// notionBlock represents a single content block in a Notion page.
// Requires custom UnmarshalJSON to extract the type-specific content.
type notionBlock struct {
ID string `json:"id"`
Type string `json:"type"`
HasChildren bool `json:"has_children"`
RawContent json.RawMessage `json:"-"` // Extracted from the type-named field (e.g., "paragraph")
Children []notionBlock `json:"-"` // Populated by client.GetBlockChildrenAll, not from API directly
}
// UnmarshalJSON implements custom JSON unmarshaling for notionBlock.
// It extracts the dynamic type-keyed field (e.g., {"type":"paragraph","paragraph":{...}})
// into RawContent.
func (b *notionBlock) UnmarshalJSON(data []byte) error {
// Use a helper struct to avoid infinite recursion
type blockAlias struct {
ID string `json:"id"`
Type string `json:"type"`
HasChildren bool `json:"has_children"`
}
var alias blockAlias
if err := json.Unmarshal(data, &alias); err != nil {
return err
}
b.ID = alias.ID
b.Type = alias.Type
b.HasChildren = alias.HasChildren
// Extract the type-named field into RawContent
if alias.Type != "" {
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
if content, ok := raw[alias.Type]; ok {
b.RawContent = content
}
}
return nil
}
// notionRichText represents a rich text element in Notion.
type notionRichText struct {
Type string `json:"type"` // "text" | "mention" | "equation"
PlainText string `json:"plain_text"`
Href string `json:"href,omitempty"`
Annotations notionAnnotations `json:"annotations"`
Text *notionTextContent `json:"text,omitempty"`
Mention *notionMention `json:"mention,omitempty"`
Equation *notionEquation `json:"equation,omitempty"`
}
// notionAnnotations holds text style information.
type notionAnnotations struct {
Bold bool `json:"bold"`
Italic bool `json:"italic"`
Strikethrough bool `json:"strikethrough"`
Underline bool `json:"underline"`
Code bool `json:"code"`
Color string `json:"color"`
}
// notionTextContent holds the content for "text" type rich text.
type notionTextContent struct {
Content string `json:"content"`
Link *struct {
URL string `json:"url"`
} `json:"link,omitempty"`
}
// notionMention holds the content for "mention" type rich text.
type notionMention struct {
Type string `json:"type"` // "page" | "database" | "date" | "user" | "link_preview"
Page *struct {
ID string `json:"id"`
} `json:"page,omitempty"`
Database *struct {
ID string `json:"id"`
} `json:"database,omitempty"`
Date *notionDateMention `json:"date,omitempty"`
LinkPreview *struct {
URL string `json:"url"`
} `json:"link_preview,omitempty"`
}
// notionDateMention holds date information for a mention.
type notionDateMention struct {
Start string `json:"start"`
End string `json:"end,omitempty"`
}
// notionEquation holds the content for "equation" type rich text.
type notionEquation struct {
Expression string `json:"expression"`
}
// notionFile represents a file object in Notion (used by image, file, pdf blocks).
type notionFile struct {
Type string `json:"type"` // "file" | "external" | "file_upload"
File *struct {
URL string `json:"url"`
ExpiryTime time.Time `json:"expiry_time"`
} `json:"file,omitempty"`
External *struct {
URL string `json:"url"`
} `json:"external,omitempty"`
FileUpload *struct {
ID string `json:"id"`
} `json:"file_upload,omitempty"`
Caption []notionRichText `json:"caption,omitempty"`
Name string `json:"name,omitempty"`
}
// GetURL returns the file URL regardless of type (hosted or external).
// For file_upload type, returns empty — caller must resolve via client.GetFileUploadURL().
func (f *notionFile) GetURL() string {
if f.File != nil {
return f.File.URL
}
if f.External != nil {
return f.External.URL
}
return ""
}
// GetFileUploadID returns the file upload ID for file_upload type, empty otherwise.
func (f *notionFile) GetFileUploadID() string {
if f.FileUpload != nil {
return f.FileUpload.ID
}
return ""
}
// --- Cursor ---
// notionCursor tracks state for incremental sync.
// Only PageEditTimes is used for diffing; LastSyncTime lives on types.SyncCursor.
type notionCursor struct {
PageEditTimes map[string]time.Time `json:"page_edit_times"` // page_id → last_edited_time
}
// --- Attachment ---
// attachment represents a file to be downloaded from a Notion page.
type attachment struct {
URL string // Notion S3 signed URL (expires in 1 hour)
FileName string
Type string // "image" | "file" | "pdf" | "video" | "audio"
}
// --- Pagination ---
// paginatedResponse is the common response wrapper for paginated Notion API responses.
type paginatedResponse struct {
Object string `json:"object"` // "list"
Results json.RawMessage `json:"results"`
HasMore bool `json:"has_more"`
NextCursor string `json:"next_cursor,omitempty"`
}
@@ -0,0 +1,144 @@
package notion
import (
"encoding/json"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
func TestParseNotionConfig(t *testing.T) {
t.Run("valid", func(t *testing.T) {
cfg, err := parseNotionConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{
"api_key": "ntn_test123",
},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if cfg.APIKey != "ntn_test123" {
t.Errorf("APIKey = %q, want %q", cfg.APIKey, "ntn_test123")
}
})
t.Run("nil config", func(t *testing.T) {
_, err := parseNotionConfig(nil)
if err == nil {
t.Fatal("expected error for nil config")
}
})
t.Run("missing api_key", func(t *testing.T) {
_, err := parseNotionConfig(&types.DataSourceConfig{
Credentials: map[string]interface{}{},
})
if err == nil {
t.Fatal("expected error for missing api_key")
}
})
}
func TestNotionBlockUnmarshalJSON(t *testing.T) {
t.Run("paragraph block", func(t *testing.T) {
data := []byte(`{
"id": "block-1",
"type": "paragraph",
"has_children": false,
"paragraph": {"rich_text": [{"type": "text", "plain_text": "hello"}]}
}`)
var block notionBlock
if err := json.Unmarshal(data, &block); err != nil {
t.Fatalf("unmarshal error: %v", err)
}
if block.ID != "block-1" {
t.Errorf("ID = %q, want %q", block.ID, "block-1")
}
if block.Type != "paragraph" {
t.Errorf("Type = %q, want %q", block.Type, "paragraph")
}
if block.HasChildren != false {
t.Error("HasChildren should be false")
}
if block.RawContent == nil {
t.Fatal("RawContent should not be nil")
}
// Verify RawContent contains the paragraph object
var content struct {
RichText []notionRichText `json:"rich_text"`
}
if err := json.Unmarshal(block.RawContent, &content); err != nil {
t.Fatalf("unmarshal RawContent: %v", err)
}
if len(content.RichText) != 1 || content.RichText[0].PlainText != "hello" {
t.Errorf("unexpected rich_text content: %+v", content.RichText)
}
})
t.Run("block with children flag", func(t *testing.T) {
data := []byte(`{
"id": "block-2",
"type": "toggle",
"has_children": true,
"toggle": {"rich_text": [{"type": "text", "plain_text": "Details"}]}
}`)
var block notionBlock
if err := json.Unmarshal(data, &block); err != nil {
t.Fatalf("unmarshal error: %v", err)
}
if !block.HasChildren {
t.Error("HasChildren should be true")
}
// Children is populated by client, not by unmarshal
if block.Children != nil {
t.Error("Children should be nil after unmarshal")
}
})
t.Run("unknown block type", func(t *testing.T) {
data := []byte(`{"id": "block-3", "type": "future_type", "has_children": false}`)
var block notionBlock
if err := json.Unmarshal(data, &block); err != nil {
t.Fatalf("unmarshal error: %v", err)
}
if block.Type != "future_type" {
t.Errorf("Type = %q", block.Type)
}
// RawContent may be nil for unknown types without matching field
})
}
func TestNotionFileGetURL(t *testing.T) {
t.Run("hosted file", func(t *testing.T) {
f := &notionFile{
Type: "file",
File: &struct {
URL string `json:"url"`
ExpiryTime time.Time `json:"expiry_time"`
}{URL: "https://s3.example.com/file.pdf"},
}
if got := f.GetURL(); got != "https://s3.example.com/file.pdf" {
t.Errorf("GetURL() = %q", got)
}
})
t.Run("external file", func(t *testing.T) {
f := &notionFile{
Type: "external",
External: &struct {
URL string `json:"url"`
}{URL: "https://example.com/image.png"},
}
if got := f.GetURL(); got != "https://example.com/image.png" {
t.Errorf("GetURL() = %q", got)
}
})
t.Run("empty file", func(t *testing.T) {
f := &notionFile{Type: "file"}
if got := f.GetURL(); got != "" {
t.Errorf("GetURL() = %q, want empty", got)
}
})
}
+3
View File
@@ -232,6 +232,9 @@ type Resource struct {
// For hierarchical resources (parent ID if applicable)
ParentID string `json:"parent_id,omitempty"`
// Whether this resource has children that can be expanded
HasChildren bool `json:"has_children,omitempty"`
// Additional metadata
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
+1
View File
@@ -28,6 +28,7 @@ const (
ChannelDingtalk = "dingtalk" // DingTalk
ChannelSlack = "slack" // Slack
ChannelIM = "im" // Generic IM channel
ChannelNotion = "notion" // Notion
)
// Knowledge parse status constants