feat(notion): add Notion API client and type definitions
- Define notionPage, notionBlock, notionParent, notionFile and cursor types - Implement rate-limited HTTP client with context-aware retry and backoff - Support search, page/block retrieval, database queries, and file downloads - Add HasChildren field to Resource type, NotionChannel to knowledge types - Add unit tests for types and client
This commit is contained in:
@@ -0,0 +1,523 @@
|
||||
package notion
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// notionClient wraps the Notion API with rate limiting and retry logic.
|
||||
type notionClient struct {
|
||||
token string
|
||||
httpClient *http.Client
|
||||
limiter *rate.Limiter
|
||||
baseURL string
|
||||
}
|
||||
|
||||
// newClient creates a new Notion API client.
|
||||
func newClient(token, baseURL string) *notionClient {
|
||||
if baseURL == "" {
|
||||
baseURL = DefaultBaseURL
|
||||
}
|
||||
return ¬ionClient{
|
||||
token: token,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
limiter: rate.NewLimiter(rate.Limit(3), 3), // 3 req/s with burst of 3 (matches Notion's actual behavior)
|
||||
baseURL: baseURL,
|
||||
}
|
||||
}
|
||||
|
||||
const maxRetries = 3
|
||||
|
||||
// sleepWithContext pauses for the given duration, returning early if ctx is cancelled.
|
||||
func sleepWithContext(ctx context.Context, d time.Duration) error {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-t.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// doRequest performs an authenticated, rate-limited HTTP request to the Notion API.
|
||||
func (c *notionClient) doRequest(ctx context.Context, method, path string, body interface{}) ([]byte, error) {
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return nil, fmt.Errorf("rate limiter: %w", err)
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
bodyBytes, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request body: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
req.Header.Set("Notion-Version", NotionAPIVersion)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
if body != nil {
|
||||
bodyBytes, _ := json.Marshal(body)
|
||||
req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return respBody, nil
|
||||
|
||||
case resp.StatusCode == 401 || resp.StatusCode == 403:
|
||||
return nil, fmt.Errorf("%w: %s", datasource.ErrInvalidCredentials, string(respBody))
|
||||
|
||||
case resp.StatusCode == 404:
|
||||
return nil, fmt.Errorf("%w: %s", datasource.ErrResourceNotFound, path)
|
||||
|
||||
case resp.StatusCode == 429:
|
||||
retryAfter := resp.Header.Get("Retry-After")
|
||||
wait := 1 * time.Second
|
||||
if secs, err := strconv.ParseFloat(retryAfter, 64); err == nil && secs > 0 {
|
||||
wait = time.Duration(secs * float64(time.Second))
|
||||
}
|
||||
logger.Warnf(ctx, "[Notion] rate limited, retry after %v (attempt %d/%d)", wait, attempt+1, maxRetries)
|
||||
lastErr = fmt.Errorf("rate limited: %s", string(respBody))
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepWithContext(ctx, wait); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
case resp.StatusCode >= 500:
|
||||
lastErr = fmt.Errorf("server error %d: %s", resp.StatusCode, string(respBody))
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf("%w: %v", datasource.ErrFetchFailed, lastErr)
|
||||
}
|
||||
return nil, datasource.ErrFetchFailed
|
||||
}
|
||||
|
||||
// Ping verifies the API token is valid by calling GET /v1/users/me.
|
||||
func (c *notionClient) Ping(ctx context.Context) error {
|
||||
_, err := c.doRequest(ctx, http.MethodGet, "/v1/users/me", nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// SearchPages returns all pages and databases accessible to the integration.
|
||||
func (c *notionClient) SearchPages(ctx context.Context) ([]notionPage, error) {
|
||||
return c.paginatePages(ctx, http.MethodPost, "/v1/search")
|
||||
}
|
||||
|
||||
// GetPage retrieves a single page by ID.
|
||||
func (c *notionClient) GetPage(ctx context.Context, pageID string) (*notionPage, error) {
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/pages/"+pageID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var page notionPage
|
||||
if err := json.Unmarshal(respBody, &page); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal page: %w", err)
|
||||
}
|
||||
page.Title = extractTitle(&page)
|
||||
return &page, nil
|
||||
}
|
||||
|
||||
// databaseInfo holds both the metadata and the data source ID from a single API call.
|
||||
type databaseInfo struct {
|
||||
Page notionPage
|
||||
DataSourceID string
|
||||
}
|
||||
|
||||
// GetDatabaseInfo retrieves a database container by ID, returning both
|
||||
// the page metadata and the primary data source ID in a single API call.
|
||||
func (c *notionClient) GetDatabaseInfo(ctx context.Context, dbID string) (*databaseInfo, error) {
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/databases/"+dbID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db notionPage
|
||||
if err := json.Unmarshal(respBody, &db); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal database: %w", err)
|
||||
}
|
||||
db.Title = extractTitle(&db)
|
||||
|
||||
var dsResult struct {
|
||||
DataSources []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data_sources"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &dsResult); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal data_sources: %w", err)
|
||||
}
|
||||
|
||||
dsID := ""
|
||||
if len(dsResult.DataSources) > 0 {
|
||||
dsID = dsResult.DataSources[0].ID
|
||||
}
|
||||
|
||||
return &databaseInfo{Page: db, DataSourceID: dsID}, nil
|
||||
}
|
||||
|
||||
// GetDataSourceInfo retrieves a data source by ID, returning its metadata.
|
||||
// In API 2025-09-03+, data_source objects hold the schema/properties and are
|
||||
// the target for record queries. The response includes database_parent to
|
||||
// locate the database in the workspace hierarchy.
|
||||
func (c *notionClient) GetDataSourceInfo(ctx context.Context, dsID string) (*notionPage, error) {
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/data_sources/"+dsID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ds notionPage
|
||||
if err := json.Unmarshal(respBody, &ds); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal data_source: %w", err)
|
||||
}
|
||||
ds.Title = extractTitle(&ds)
|
||||
return &ds, nil
|
||||
}
|
||||
|
||||
// GetBlockChildrenFlat fetches only the direct children of a block (no recursion).
|
||||
// Used by discoverPages to quickly scan for child_page/child_database without
|
||||
// fetching the full block tree content.
|
||||
func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string) ([]notionBlock, error) {
|
||||
var allBlocks []notionBlock
|
||||
var startCursor string
|
||||
|
||||
for {
|
||||
path := fmt.Sprintf("/v1/blocks/%s/children", blockID)
|
||||
if startCursor != "" {
|
||||
path += "?start_cursor=" + startCursor
|
||||
}
|
||||
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get block children for %s: %w", blockID, err)
|
||||
}
|
||||
|
||||
var resp paginatedResponse
|
||||
if err := json.Unmarshal(respBody, &resp); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal block children response: %w", err)
|
||||
}
|
||||
|
||||
var blocks []notionBlock
|
||||
if err := json.Unmarshal(resp.Results, &blocks); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal blocks: %w", err)
|
||||
}
|
||||
|
||||
allBlocks = append(allBlocks, blocks...)
|
||||
|
||||
if !resp.HasMore || resp.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
startCursor = resp.NextCursor
|
||||
}
|
||||
|
||||
return allBlocks, nil
|
||||
}
|
||||
|
||||
const maxBlockDepth = 5 // Limit recursion depth — deeper content has diminishing value for knowledge bases
|
||||
const maxBlocksPerPage = 1000 // Limit total blocks fetched per page to prevent runaway API calls
|
||||
|
||||
// GetBlockChildrenAll recursively fetches all blocks under a given block ID,
|
||||
// building a tree structure with Children populated for blocks with has_children=true.
|
||||
// child_page and child_database blocks are NOT recursed into (handled by connector layer).
|
||||
// Recursion is limited to maxBlockDepth to prevent excessive API calls on complex pages.
|
||||
func (c *notionClient) GetBlockChildrenAll(ctx context.Context, blockID string) ([]notionBlock, error) {
|
||||
return c.getBlockChildrenRecursive(ctx, blockID, 0)
|
||||
}
|
||||
|
||||
func (c *notionClient) getBlockChildrenRecursive(ctx context.Context, blockID string, depth int) ([]notionBlock, error) {
|
||||
var allBlocks []notionBlock
|
||||
var startCursor string
|
||||
|
||||
for {
|
||||
path := fmt.Sprintf("/v1/blocks/%s/children", blockID)
|
||||
if startCursor != "" {
|
||||
path += "?start_cursor=" + startCursor
|
||||
}
|
||||
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get block children for %s: %w", blockID, err)
|
||||
}
|
||||
|
||||
var resp paginatedResponse
|
||||
if err := json.Unmarshal(respBody, &resp); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal block children response: %w", err)
|
||||
}
|
||||
|
||||
var blocks []notionBlock
|
||||
if err := json.Unmarshal(resp.Results, &blocks); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal blocks: %w", err)
|
||||
}
|
||||
|
||||
allBlocks = append(allBlocks, blocks...)
|
||||
|
||||
if len(allBlocks) >= maxBlocksPerPage || !resp.HasMore || resp.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
startCursor = resp.NextCursor
|
||||
}
|
||||
|
||||
if depth >= maxBlockDepth {
|
||||
return allBlocks, nil
|
||||
}
|
||||
|
||||
for i := range allBlocks {
|
||||
if !allBlocks[i].HasChildren {
|
||||
continue
|
||||
}
|
||||
// Skip block types that don't contribute useful content
|
||||
switch allBlocks[i].Type {
|
||||
case "child_page", "child_database", "unsupported", "template", "breadcrumb", "table_of_contents":
|
||||
continue
|
||||
}
|
||||
children, err := c.getBlockChildrenRecursive(ctx, allBlocks[i].ID, depth+1)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Notion] failed to get children for block %s (depth %d): %v", allBlocks[i].ID, depth, err)
|
||||
continue
|
||||
}
|
||||
allBlocks[i].Children = children
|
||||
}
|
||||
|
||||
return allBlocks, nil
|
||||
}
|
||||
|
||||
// QueryDatabaseAll retrieves all records from a database via POST /v1/data_sources/{id}/query.
|
||||
// Accepts either a data_source_id (from search) or a database_id (from child_database blocks).
|
||||
// For database_ids, resolves to data_source_id via GET /v1/databases/{id}.
|
||||
func (c *notionClient) QueryDatabaseAll(ctx context.Context, id string) ([]notionPage, error) {
|
||||
// Try as data_source_id directly
|
||||
records, err := c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", id))
|
||||
if err == nil {
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// If 404, id might be a database container ID — resolve to data_source_id
|
||||
info, dbErr := c.GetDatabaseInfo(ctx, id)
|
||||
if dbErr != nil {
|
||||
return nil, fmt.Errorf("query database %s: not a data_source (%v) and not a database (%v)", id, err, dbErr)
|
||||
}
|
||||
if info.DataSourceID == "" {
|
||||
return nil, fmt.Errorf("database %s has no data sources", id)
|
||||
}
|
||||
return c.paginatePages(ctx, http.MethodPost, fmt.Sprintf("/v1/data_sources/%s/query", info.DataSourceID))
|
||||
}
|
||||
|
||||
// ResolveBlock re-fetches a single block to resolve file_upload URLs.
|
||||
// When a block contains a file_upload type, re-fetching it returns the resolved
|
||||
// download URL (temporary S3 signed URL, 1-hour expiry).
|
||||
func (c *notionClient) ResolveBlock(ctx context.Context, blockID string) (*notionBlock, error) {
|
||||
respBody, err := c.doRequest(ctx, http.MethodGet, "/v1/blocks/"+blockID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var block notionBlock
|
||||
if err := json.Unmarshal(respBody, &block); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal block: %w", err)
|
||||
}
|
||||
return &block, nil
|
||||
}
|
||||
|
||||
const maxDownloadSize = 100 * 1024 * 1024 // 100MB — prevent OOM from oversized files
|
||||
|
||||
// DownloadFile downloads a file from the given URL (typically an S3 signed URL).
|
||||
// Does not go through the rate limiter since it's not a Notion API call.
|
||||
func (c *notionClient) DownloadFile(ctx context.Context, fileURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if attempt < maxRetries {
|
||||
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("download failed with status %d", resp.StatusCode)
|
||||
if resp.StatusCode >= 500 && attempt < maxRetries {
|
||||
if sErr := sleepWithContext(ctx, time.Duration(1<<attempt)*time.Second); sErr != nil {
|
||||
return nil, sErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxDownloadSize+1))
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxDownloadSize {
|
||||
return nil, fmt.Errorf("file exceeds maximum download size (%d MB)", maxDownloadSize/(1024*1024))
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("download file: %w", lastErr)
|
||||
}
|
||||
|
||||
// --- Shared pagination helper ---
|
||||
|
||||
// paginatePages fetches all pages from a paginated Notion API endpoint.
|
||||
func (c *notionClient) paginatePages(ctx context.Context, method, path string) ([]notionPage, error) {
|
||||
var allPages []notionPage
|
||||
var startCursor string
|
||||
|
||||
for {
|
||||
body := map[string]interface{}{
|
||||
"page_size": 100,
|
||||
}
|
||||
if startCursor != "" {
|
||||
body["start_cursor"] = startCursor
|
||||
}
|
||||
|
||||
var respBody []byte
|
||||
var err error
|
||||
if method == http.MethodPost {
|
||||
respBody, err = c.doRequest(ctx, method, path, body)
|
||||
} else {
|
||||
p := path
|
||||
if startCursor != "" {
|
||||
p += "?start_cursor=" + startCursor + "&page_size=100"
|
||||
}
|
||||
respBody, err = c.doRequest(ctx, method, p, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("paginate %s: %w", path, err)
|
||||
}
|
||||
|
||||
var resp paginatedResponse
|
||||
if err := json.Unmarshal(respBody, &resp); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal paginated response: %w", err)
|
||||
}
|
||||
|
||||
var pages []notionPage
|
||||
if err := json.Unmarshal(resp.Results, &pages); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal page results: %w", err)
|
||||
}
|
||||
|
||||
for i := range pages {
|
||||
pages[i].Title = extractTitle(&pages[i])
|
||||
}
|
||||
|
||||
allPages = append(allPages, pages...)
|
||||
|
||||
if !resp.HasMore || resp.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
startCursor = resp.NextCursor
|
||||
}
|
||||
|
||||
return allPages, nil
|
||||
}
|
||||
|
||||
// --- Title extraction helpers ---
|
||||
|
||||
func extractTitle(page *notionPage) string {
|
||||
// Try extracting from properties (works for pages)
|
||||
if page.RawProperties != nil {
|
||||
var props map[string]json.RawMessage
|
||||
if err := json.Unmarshal(page.RawProperties, &props); err == nil {
|
||||
for _, propRaw := range props {
|
||||
var prop struct {
|
||||
Type string `json:"type"`
|
||||
Title []struct {
|
||||
PlainText string `json:"plain_text"`
|
||||
} `json:"title"`
|
||||
}
|
||||
if err := json.Unmarshal(propRaw, &prop); err != nil {
|
||||
continue
|
||||
}
|
||||
if prop.Type == "title" && len(prop.Title) > 0 {
|
||||
return joinPlainText(prop.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: top-level title array (works for databases)
|
||||
if page.RawTitle != nil {
|
||||
var titleSegments []struct {
|
||||
PlainText string `json:"plain_text"`
|
||||
}
|
||||
if err := json.Unmarshal(page.RawTitle, &titleSegments); err == nil && len(titleSegments) > 0 {
|
||||
return joinPlainText(titleSegments)
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func joinPlainText(segments []struct {
|
||||
PlainText string `json:"plain_text"`
|
||||
}) string {
|
||||
var sb strings.Builder
|
||||
for _, s := range segments {
|
||||
sb.WriteString(s.PlainText)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package notion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeNotion builds an httptest.Server that emulates relevant Notion API endpoints.
|
||||
func fakeNotion() (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// GET /v1/users/me — auth check
|
||||
mux.HandleFunc("/v1/users/me", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||
w.WriteHeader(401)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": 401,
|
||||
"code": "unauthorized",
|
||||
"message": "API token is invalid.",
|
||||
})
|
||||
return
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"object": "user",
|
||||
"id": "bot-user-id",
|
||||
"type": "bot",
|
||||
"name": "Test Integration",
|
||||
})
|
||||
})
|
||||
|
||||
// POST /v1/search — return pages and databases
|
||||
mux.HandleFunc("/v1/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"object": "list",
|
||||
"results": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": "page-1",
|
||||
"object": "page",
|
||||
"url": "https://notion.so/Page-1",
|
||||
"last_edited_time": "2026-01-15T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
|
||||
"properties": map[string]interface{}{
|
||||
"title": map[string]interface{}{
|
||||
"type": "title",
|
||||
"title": []interface{}{
|
||||
map[string]interface{}{"plain_text": "Test Page"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"id": "db-1",
|
||||
"object": "data_source",
|
||||
"url": "https://notion.so/DB-1",
|
||||
"last_edited_time": "2026-01-16T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
|
||||
"title": []interface{}{
|
||||
map[string]interface{}{"plain_text": "Test Database"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"has_more": false,
|
||||
"next_cursor": nil,
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/blocks/{id}/children — return blocks
|
||||
mux.HandleFunc("/v1/blocks/page-1/children", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"object": "list",
|
||||
"results": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": "blk-1",
|
||||
"type": "paragraph",
|
||||
"has_children": false,
|
||||
"paragraph": map[string]interface{}{
|
||||
"rich_text": []interface{}{
|
||||
map[string]interface{}{
|
||||
"type": "text",
|
||||
"plain_text": "Hello world",
|
||||
"text": map[string]interface{}{"content": "Hello world"},
|
||||
"annotations": map[string]interface{}{
|
||||
"bold": false, "italic": false, "strikethrough": false,
|
||||
"underline": false, "code": false, "color": "default",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"id": "blk-2",
|
||||
"type": "child_page",
|
||||
"has_children": true,
|
||||
"child_page": map[string]interface{}{"title": "Sub Page"},
|
||||
},
|
||||
},
|
||||
"has_more": false,
|
||||
"next_cursor": nil,
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/pages/{id}
|
||||
mux.HandleFunc("/v1/pages/page-1", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": "page-1",
|
||||
"object": "page",
|
||||
"url": "https://notion.so/Page-1",
|
||||
"last_edited_time": "2026-01-15T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
|
||||
"properties": map[string]interface{}{
|
||||
"title": map[string]interface{}{
|
||||
"type": "title",
|
||||
"title": []interface{}{
|
||||
map[string]interface{}{"plain_text": "Test Page"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/databases/{id} — returns container with data_sources array (2025-09-03+)
|
||||
mux.HandleFunc("/v1/databases/db-1", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": "db-1",
|
||||
"object": "database",
|
||||
"url": "https://notion.so/DB-1",
|
||||
"last_edited_time": "2026-01-16T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "workspace", "workspace": true},
|
||||
"title": []interface{}{map[string]interface{}{"plain_text": "Test Database"}},
|
||||
"data_sources": []interface{}{
|
||||
map[string]interface{}{"id": "ds-1", "name": "Default"},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/data_sources/{id} — returns schema/properties
|
||||
mux.HandleFunc("/v1/data_sources/ds-1", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": "ds-1",
|
||||
"object": "data_source",
|
||||
"properties": map[string]interface{}{
|
||||
"Name": map[string]interface{}{"type": "title", "title": map[string]interface{}{}},
|
||||
"Status": map[string]interface{}{"type": "select", "select": map[string]interface{}{}},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
// POST /v1/data_sources/{id}/query — return database records (2025-09-03+)
|
||||
mux.HandleFunc("/v1/data_sources/ds-1/query", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"object": "list",
|
||||
"results": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": "record-1",
|
||||
"object": "page",
|
||||
"url": "https://notion.so/Record-1",
|
||||
"last_edited_time": "2026-01-17T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"},
|
||||
"properties": map[string]interface{}{
|
||||
"Name": map[string]interface{}{
|
||||
"type": "title",
|
||||
"title": []interface{}{
|
||||
map[string]interface{}{"plain_text": "Record One"},
|
||||
},
|
||||
},
|
||||
"Status": map[string]interface{}{
|
||||
"type": "select",
|
||||
"select": map[string]interface{}{"name": "Done"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"has_more": false,
|
||||
"next_cursor": nil,
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/pages/record-1 — direct fetch of a database row (single-select case).
|
||||
// fetchPage must recognize both database_id (older responses) and
|
||||
// data_source_id (2025-09-03+) as a record and route to buildRecordItem.
|
||||
mux.HandleFunc("/v1/pages/record-1", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": "record-1",
|
||||
"object": "page",
|
||||
"url": "https://notion.so/Record-1",
|
||||
"last_edited_time": "2026-01-17T10:00:00.000Z",
|
||||
"in_trash": false,
|
||||
"parent": map[string]interface{}{"type": "data_source_id", "data_source_id": "ds-1"},
|
||||
"properties": map[string]interface{}{
|
||||
"Name": map[string]interface{}{
|
||||
"type": "title",
|
||||
"title": []interface{}{
|
||||
map[string]interface{}{"plain_text": "Record One"},
|
||||
},
|
||||
},
|
||||
"Status": map[string]interface{}{
|
||||
"type": "select",
|
||||
"select": map[string]interface{}{"name": "Done"},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
// GET /v1/blocks/record-1/children — empty blocks for database record
|
||||
mux.HandleFunc("/v1/blocks/record-1/children", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"object": "list",
|
||||
"results": []interface{}{},
|
||||
"has_more": false,
|
||||
"next_cursor": nil,
|
||||
})
|
||||
})
|
||||
|
||||
ts := httptest.NewServer(mux)
|
||||
cfg := &Config{APIKey: "test-token"}
|
||||
return ts, cfg
|
||||
}
|
||||
|
||||
func TestClientPing(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
if err := client.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("Ping() error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientPing_InvalidToken(t *testing.T) {
|
||||
ts, _ := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient("wrong-token", ts.URL)
|
||||
err := client.Ping(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientSearchPages(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
pages, err := client.SearchPages(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("SearchPages() error: %v", err)
|
||||
}
|
||||
if len(pages) != 2 {
|
||||
t.Fatalf("expected 2 pages, got %d", len(pages))
|
||||
}
|
||||
// First result is a page
|
||||
if pages[0].Object != "page" || pages[0].ID != "page-1" {
|
||||
t.Errorf("page[0] = %+v", pages[0])
|
||||
}
|
||||
// Second result is a data_source (databases are returned as object="data_source"
|
||||
// in API 2025-09-03+ which is what the mock simulates)
|
||||
if pages[1].Object != "data_source" || pages[1].ID != "db-1" {
|
||||
t.Errorf("page[1] = %+v", pages[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientGetBlockChildrenAll(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
blocks, err := client.GetBlockChildrenAll(context.Background(), "page-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetBlockChildrenAll() error: %v", err)
|
||||
}
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("expected 2 blocks, got %d", len(blocks))
|
||||
}
|
||||
if blocks[0].Type != "paragraph" {
|
||||
t.Errorf("block[0].Type = %q", blocks[0].Type)
|
||||
}
|
||||
if blocks[1].Type != "child_page" {
|
||||
t.Errorf("block[1].Type = %q", blocks[1].Type)
|
||||
}
|
||||
// child_page should NOT have its children recursed
|
||||
if blocks[1].Children != nil {
|
||||
t.Error("child_page children should not be recursed by client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientGetPage(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
page, err := client.GetPage(context.Background(), "page-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetPage() error: %v", err)
|
||||
}
|
||||
if page.ID != "page-1" {
|
||||
t.Errorf("ID = %q", page.ID)
|
||||
}
|
||||
if page.Title != "Test Page" {
|
||||
t.Errorf("Title = %q, want %q", page.Title, "Test Page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientQueryDatabaseAll(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
records, err := client.QueryDatabaseAll(context.Background(), "db-1")
|
||||
if err != nil {
|
||||
t.Fatalf("QueryDatabaseAll() error: %v", err)
|
||||
}
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(records))
|
||||
}
|
||||
if records[0].ID != "record-1" {
|
||||
t.Errorf("record ID = %q", records[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure time import is used (referenced by fakeNotion indirectly via time.Time in types).
|
||||
var _ = time.Now
|
||||
@@ -0,0 +1,273 @@
|
||||
// Package notion implements the Notion data source connector for WeKnora.
|
||||
//
|
||||
// It syncs pages, databases, and attachments from Notion workspaces into WeKnora knowledge bases.
|
||||
//
|
||||
// Notion API docs:
|
||||
// - Authentication: https://developers.notion.com/docs/authorization
|
||||
// - Search: https://developers.notion.com/reference/post-search
|
||||
// - Pages: https://developers.notion.com/reference/retrieve-a-page
|
||||
// - Blocks: https://developers.notion.com/reference/retrieve-a-block
|
||||
// - Databases: https://developers.notion.com/reference/retrieve-a-database
|
||||
package notion
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// NotionAPIVersion is the Notion API version used by this connector.
|
||||
const NotionAPIVersion = "2026-03-11"
|
||||
|
||||
// DefaultBaseURL is the Notion API base URL.
|
||||
const DefaultBaseURL = "https://api.notion.com"
|
||||
|
||||
// Config holds Notion-specific configuration for the data source connector.
|
||||
type Config struct {
|
||||
APIKey string `json:"api_key"` // Internal Integration Token
|
||||
}
|
||||
|
||||
// parseNotionConfig extracts and validates Notion config from DataSourceConfig.
|
||||
func parseNotionConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
if config == nil {
|
||||
return nil, datasource.ErrInvalidConfig
|
||||
}
|
||||
|
||||
tokenVal, ok := config.Credentials["api_key"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: missing api_key", datasource.ErrInvalidCredentials)
|
||||
}
|
||||
token, ok := tokenVal.(string)
|
||||
if !ok || token == "" {
|
||||
return nil, fmt.Errorf("%w: api_key must be a non-empty string", datasource.ErrInvalidCredentials)
|
||||
}
|
||||
|
||||
return &Config{APIKey: token}, nil
|
||||
}
|
||||
|
||||
// --- API response types ---
|
||||
|
||||
// notionPage represents a Notion page or database object.
|
||||
type notionPage struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"` // "page" | "database" | "data_source" (2025-09-03+)
|
||||
Parent notionParent `json:"parent"`
|
||||
URL string `json:"url"`
|
||||
LastEditedTime time.Time `json:"last_edited_time"`
|
||||
InTrash bool `json:"in_trash"`
|
||||
// Title is extracted from properties after unmarshaling; not directly from JSON.
|
||||
Title string `json:"-"`
|
||||
// RawTitle holds the top-level title array (used by database objects).
|
||||
RawTitle json.RawMessage `json:"title,omitempty"`
|
||||
// RawProperties holds the raw JSON for property extraction.
|
||||
RawProperties json.RawMessage `json:"properties"`
|
||||
// DatabaseParent indicates where the database lives in the workspace hierarchy.
|
||||
// Only present on data_source objects (API 2025-09-03+). For example, if a database
|
||||
// is inside a page, this will be {type: "page_id", page_id: "..."}.
|
||||
DatabaseParent *notionParent `json:"database_parent,omitempty"`
|
||||
}
|
||||
|
||||
// Parent type constants for notionParent.Type
|
||||
const (
|
||||
parentTypeWorkspace = "workspace"
|
||||
parentTypePageID = "page_id"
|
||||
parentTypeDatabaseID = "database_id"
|
||||
parentTypeDataSourceID = "data_source_id"
|
||||
parentTypeBlockID = "block_id"
|
||||
)
|
||||
|
||||
// notionParent represents the parent relationship of a page/database.
|
||||
type notionParent struct {
|
||||
Type string `json:"type"`
|
||||
PageID string `json:"page_id,omitempty"`
|
||||
DatabaseID string `json:"database_id,omitempty"`
|
||||
DataSourceID string `json:"data_source_id,omitempty"`
|
||||
BlockID string `json:"block_id,omitempty"`
|
||||
}
|
||||
|
||||
// GetParentID returns the effective parent ID regardless of parent type.
|
||||
func (p *notionParent) GetParentID() string {
|
||||
switch p.Type {
|
||||
case parentTypePageID:
|
||||
return p.PageID
|
||||
case parentTypeDatabaseID:
|
||||
return p.DatabaseID
|
||||
case parentTypeDataSourceID:
|
||||
return p.DataSourceID
|
||||
case parentTypeBlockID:
|
||||
return p.BlockID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// isDatabase returns true if the notionPage represents a database or data source.
|
||||
func (p *notionPage) isDatabase() bool {
|
||||
return p.Object == "database" || p.Object == "data_source"
|
||||
}
|
||||
|
||||
// notionBlock represents a single content block in a Notion page.
|
||||
// Requires custom UnmarshalJSON to extract the type-specific content.
|
||||
type notionBlock struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
HasChildren bool `json:"has_children"`
|
||||
RawContent json.RawMessage `json:"-"` // Extracted from the type-named field (e.g., "paragraph")
|
||||
Children []notionBlock `json:"-"` // Populated by client.GetBlockChildrenAll, not from API directly
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements custom JSON unmarshaling for notionBlock.
|
||||
// It extracts the dynamic type-keyed field (e.g., {"type":"paragraph","paragraph":{...}})
|
||||
// into RawContent.
|
||||
func (b *notionBlock) UnmarshalJSON(data []byte) error {
|
||||
// Use a helper struct to avoid infinite recursion
|
||||
type blockAlias struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
HasChildren bool `json:"has_children"`
|
||||
}
|
||||
|
||||
var alias blockAlias
|
||||
if err := json.Unmarshal(data, &alias); err != nil {
|
||||
return err
|
||||
}
|
||||
b.ID = alias.ID
|
||||
b.Type = alias.Type
|
||||
b.HasChildren = alias.HasChildren
|
||||
|
||||
// Extract the type-named field into RawContent
|
||||
if alias.Type != "" {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if content, ok := raw[alias.Type]; ok {
|
||||
b.RawContent = content
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// notionRichText represents a rich text element in Notion.
|
||||
type notionRichText struct {
|
||||
Type string `json:"type"` // "text" | "mention" | "equation"
|
||||
PlainText string `json:"plain_text"`
|
||||
Href string `json:"href,omitempty"`
|
||||
Annotations notionAnnotations `json:"annotations"`
|
||||
Text *notionTextContent `json:"text,omitempty"`
|
||||
Mention *notionMention `json:"mention,omitempty"`
|
||||
Equation *notionEquation `json:"equation,omitempty"`
|
||||
}
|
||||
|
||||
// notionAnnotations holds text style information.
|
||||
type notionAnnotations struct {
|
||||
Bold bool `json:"bold"`
|
||||
Italic bool `json:"italic"`
|
||||
Strikethrough bool `json:"strikethrough"`
|
||||
Underline bool `json:"underline"`
|
||||
Code bool `json:"code"`
|
||||
Color string `json:"color"`
|
||||
}
|
||||
|
||||
// notionTextContent holds the content for "text" type rich text.
|
||||
type notionTextContent struct {
|
||||
Content string `json:"content"`
|
||||
Link *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"link,omitempty"`
|
||||
}
|
||||
|
||||
// notionMention holds the content for "mention" type rich text.
|
||||
type notionMention struct {
|
||||
Type string `json:"type"` // "page" | "database" | "date" | "user" | "link_preview"
|
||||
Page *struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"page,omitempty"`
|
||||
Database *struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"database,omitempty"`
|
||||
Date *notionDateMention `json:"date,omitempty"`
|
||||
LinkPreview *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"link_preview,omitempty"`
|
||||
}
|
||||
|
||||
// notionDateMention holds date information for a mention.
|
||||
type notionDateMention struct {
|
||||
Start string `json:"start"`
|
||||
End string `json:"end,omitempty"`
|
||||
}
|
||||
|
||||
// notionEquation holds the content for "equation" type rich text.
|
||||
type notionEquation struct {
|
||||
Expression string `json:"expression"`
|
||||
}
|
||||
|
||||
// notionFile represents a file object in Notion (used by image, file, pdf blocks).
|
||||
type notionFile struct {
|
||||
Type string `json:"type"` // "file" | "external" | "file_upload"
|
||||
File *struct {
|
||||
URL string `json:"url"`
|
||||
ExpiryTime time.Time `json:"expiry_time"`
|
||||
} `json:"file,omitempty"`
|
||||
External *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"external,omitempty"`
|
||||
FileUpload *struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"file_upload,omitempty"`
|
||||
Caption []notionRichText `json:"caption,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// GetURL returns the file URL regardless of type (hosted or external).
|
||||
// For file_upload type, returns empty — caller must resolve via client.GetFileUploadURL().
|
||||
func (f *notionFile) GetURL() string {
|
||||
if f.File != nil {
|
||||
return f.File.URL
|
||||
}
|
||||
if f.External != nil {
|
||||
return f.External.URL
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetFileUploadID returns the file upload ID for file_upload type, empty otherwise.
|
||||
func (f *notionFile) GetFileUploadID() string {
|
||||
if f.FileUpload != nil {
|
||||
return f.FileUpload.ID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// --- Cursor ---
|
||||
|
||||
// notionCursor tracks state for incremental sync.
|
||||
// Only PageEditTimes is used for diffing; LastSyncTime lives on types.SyncCursor.
|
||||
type notionCursor struct {
|
||||
PageEditTimes map[string]time.Time `json:"page_edit_times"` // page_id → last_edited_time
|
||||
}
|
||||
|
||||
// --- Attachment ---
|
||||
|
||||
// attachment represents a file to be downloaded from a Notion page.
|
||||
type attachment struct {
|
||||
URL string // Notion S3 signed URL (expires in 1 hour)
|
||||
FileName string
|
||||
Type string // "image" | "file" | "pdf" | "video" | "audio"
|
||||
}
|
||||
|
||||
// --- Pagination ---
|
||||
|
||||
// paginatedResponse is the common response wrapper for paginated Notion API responses.
|
||||
type paginatedResponse struct {
|
||||
Object string `json:"object"` // "list"
|
||||
Results json.RawMessage `json:"results"`
|
||||
HasMore bool `json:"has_more"`
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package notion
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestParseNotionConfig(t *testing.T) {
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
cfg, err := parseNotionConfig(&types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{
|
||||
"api_key": "ntn_test123",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if cfg.APIKey != "ntn_test123" {
|
||||
t.Errorf("APIKey = %q, want %q", cfg.APIKey, "ntn_test123")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil config", func(t *testing.T) {
|
||||
_, err := parseNotionConfig(nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for nil config")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing api_key", func(t *testing.T) {
|
||||
_, err := parseNotionConfig(&types.DataSourceConfig{
|
||||
Credentials: map[string]interface{}{},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing api_key")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNotionBlockUnmarshalJSON(t *testing.T) {
|
||||
t.Run("paragraph block", func(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"id": "block-1",
|
||||
"type": "paragraph",
|
||||
"has_children": false,
|
||||
"paragraph": {"rich_text": [{"type": "text", "plain_text": "hello"}]}
|
||||
}`)
|
||||
var block notionBlock
|
||||
if err := json.Unmarshal(data, &block); err != nil {
|
||||
t.Fatalf("unmarshal error: %v", err)
|
||||
}
|
||||
if block.ID != "block-1" {
|
||||
t.Errorf("ID = %q, want %q", block.ID, "block-1")
|
||||
}
|
||||
if block.Type != "paragraph" {
|
||||
t.Errorf("Type = %q, want %q", block.Type, "paragraph")
|
||||
}
|
||||
if block.HasChildren != false {
|
||||
t.Error("HasChildren should be false")
|
||||
}
|
||||
if block.RawContent == nil {
|
||||
t.Fatal("RawContent should not be nil")
|
||||
}
|
||||
// Verify RawContent contains the paragraph object
|
||||
var content struct {
|
||||
RichText []notionRichText `json:"rich_text"`
|
||||
}
|
||||
if err := json.Unmarshal(block.RawContent, &content); err != nil {
|
||||
t.Fatalf("unmarshal RawContent: %v", err)
|
||||
}
|
||||
if len(content.RichText) != 1 || content.RichText[0].PlainText != "hello" {
|
||||
t.Errorf("unexpected rich_text content: %+v", content.RichText)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("block with children flag", func(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"id": "block-2",
|
||||
"type": "toggle",
|
||||
"has_children": true,
|
||||
"toggle": {"rich_text": [{"type": "text", "plain_text": "Details"}]}
|
||||
}`)
|
||||
var block notionBlock
|
||||
if err := json.Unmarshal(data, &block); err != nil {
|
||||
t.Fatalf("unmarshal error: %v", err)
|
||||
}
|
||||
if !block.HasChildren {
|
||||
t.Error("HasChildren should be true")
|
||||
}
|
||||
// Children is populated by client, not by unmarshal
|
||||
if block.Children != nil {
|
||||
t.Error("Children should be nil after unmarshal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown block type", func(t *testing.T) {
|
||||
data := []byte(`{"id": "block-3", "type": "future_type", "has_children": false}`)
|
||||
var block notionBlock
|
||||
if err := json.Unmarshal(data, &block); err != nil {
|
||||
t.Fatalf("unmarshal error: %v", err)
|
||||
}
|
||||
if block.Type != "future_type" {
|
||||
t.Errorf("Type = %q", block.Type)
|
||||
}
|
||||
// RawContent may be nil for unknown types without matching field
|
||||
})
|
||||
}
|
||||
|
||||
func TestNotionFileGetURL(t *testing.T) {
|
||||
t.Run("hosted file", func(t *testing.T) {
|
||||
f := ¬ionFile{
|
||||
Type: "file",
|
||||
File: &struct {
|
||||
URL string `json:"url"`
|
||||
ExpiryTime time.Time `json:"expiry_time"`
|
||||
}{URL: "https://s3.example.com/file.pdf"},
|
||||
}
|
||||
if got := f.GetURL(); got != "https://s3.example.com/file.pdf" {
|
||||
t.Errorf("GetURL() = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("external file", func(t *testing.T) {
|
||||
f := ¬ionFile{
|
||||
Type: "external",
|
||||
External: &struct {
|
||||
URL string `json:"url"`
|
||||
}{URL: "https://example.com/image.png"},
|
||||
}
|
||||
if got := f.GetURL(); got != "https://example.com/image.png" {
|
||||
t.Errorf("GetURL() = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty file", func(t *testing.T) {
|
||||
f := ¬ionFile{Type: "file"}
|
||||
if got := f.GetURL(); got != "" {
|
||||
t.Errorf("GetURL() = %q, want empty", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -232,6 +232,9 @@ type Resource struct {
|
||||
// For hierarchical resources (parent ID if applicable)
|
||||
ParentID string `json:"parent_id,omitempty"`
|
||||
|
||||
// Whether this resource has children that can be expanded
|
||||
HasChildren bool `json:"has_children,omitempty"`
|
||||
|
||||
// Additional metadata
|
||||
Metadata map[string]interface{} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ const (
|
||||
ChannelDingtalk = "dingtalk" // DingTalk
|
||||
ChannelSlack = "slack" // Slack
|
||||
ChannelIM = "im" // Generic IM channel
|
||||
ChannelNotion = "notion" // Notion
|
||||
)
|
||||
|
||||
// Knowledge parse status constants
|
||||
|
||||
Reference in New Issue
Block a user