fix: update environment configuration for debugging and logging, enhance SQLite FTS5 table management, and improve API base URL handling
This commit is contained in:
@@ -394,9 +394,6 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id
|
||||
if err != nil || len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Same database may be reached via two different IDs (data_source_id from search
|
||||
// vs database_container_id from child_database blocks). Mark the canonical
|
||||
// data_source_id as visited so the second path is deduped.
|
||||
if queryID != "" && queryID != id {
|
||||
if visited[queryID] {
|
||||
return nil
|
||||
@@ -404,18 +401,15 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id
|
||||
visited[queryID] = true
|
||||
}
|
||||
|
||||
propNames := extractPropertySchema(records[0])
|
||||
var items []types.FetchedItem
|
||||
for _, record := range records {
|
||||
if record.InTrash {
|
||||
continue
|
||||
}
|
||||
item := c.buildRecordItem(ctx, client, record, propNames, dbTitle)
|
||||
if item != nil {
|
||||
items = append(items, *item)
|
||||
}
|
||||
visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls
|
||||
}
|
||||
return items
|
||||
|
||||
item := c.buildDatabaseItem(ctx, client, id, dbTitle, records)
|
||||
if item != nil {
|
||||
return []types.FetchedItem{*item}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchDatabaseIncremental syncs only changed records by comparing edit times against cursor.
|
||||
@@ -438,32 +432,33 @@ func (c *Connector) fetchDatabaseIncremental(ctx context.Context, client *notion
|
||||
}
|
||||
|
||||
recordEditTimes := make(map[string]time.Time, len(records))
|
||||
var propNames []string
|
||||
if len(records) > 0 {
|
||||
propNames = extractPropertySchema(records[0])
|
||||
}
|
||||
|
||||
var items []types.FetchedItem
|
||||
changedCount := 0
|
||||
|
||||
for _, record := range records {
|
||||
visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls
|
||||
|
||||
if record.InTrash {
|
||||
continue
|
||||
}
|
||||
recordEditTimes[record.ID] = record.LastEditedTime
|
||||
|
||||
prevTime, existed := prevEditTimes[record.ID]
|
||||
if existed && record.LastEditedTime.Equal(prevTime) {
|
||||
continue
|
||||
}
|
||||
changedCount++
|
||||
item := c.buildRecordItem(ctx, client, record, propNames, dbTitle)
|
||||
if item != nil {
|
||||
items = append(items, *item)
|
||||
if !existed || !record.LastEditedTime.Equal(prevTime) {
|
||||
changedCount++
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Notion] database %s incremental: %d changed out of %d records", id, changedCount, len(records))
|
||||
return items, recordEditTimes
|
||||
|
||||
// Rebuild the entire database table if any record changed, or if we don't have previous times (first sync)
|
||||
if changedCount > 0 || len(prevEditTimes) == 0 {
|
||||
item := c.buildDatabaseItem(ctx, client, id, dbTitle, records)
|
||||
if item != nil {
|
||||
return []types.FetchedItem{*item}, recordEditTimes
|
||||
}
|
||||
}
|
||||
|
||||
return nil, recordEditTimes
|
||||
}
|
||||
|
||||
// queryDatabaseRecords resolves the database ID and queries all records.
|
||||
@@ -545,6 +540,103 @@ func (c *Connector) buildRecordItem(ctx context.Context, client *notionClient, r
|
||||
}
|
||||
}
|
||||
|
||||
// buildDatabaseItem converts a database into a single Markdown table document.
|
||||
func (c *Connector) buildDatabaseItem(ctx context.Context, client *notionClient, id string, dbTitle string, records []notionPage) *types.FetchedItem {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
propNames := extractPropertySchema(records[0])
|
||||
|
||||
var content strings.Builder
|
||||
title := dbTitle
|
||||
if title == "" {
|
||||
title = defaultUntitledName
|
||||
}
|
||||
content.WriteString("# " + title + "\n\n")
|
||||
|
||||
// Table header
|
||||
content.WriteString("| Title ")
|
||||
for _, name := range propNames {
|
||||
content.WriteString("| " + strings.ReplaceAll(name, "|", "\\|") + " ")
|
||||
}
|
||||
content.WriteString("|\n|")
|
||||
content.WriteString("---|")
|
||||
for range propNames {
|
||||
content.WriteString("---|")
|
||||
}
|
||||
content.WriteString("\n")
|
||||
|
||||
var extraContent strings.Builder
|
||||
|
||||
// Table rows
|
||||
for _, record := range records {
|
||||
if record.InTrash {
|
||||
continue
|
||||
}
|
||||
|
||||
recordTitle := record.Title
|
||||
if recordTitle == "" {
|
||||
recordTitle = defaultUntitledName
|
||||
}
|
||||
|
||||
content.WriteString("| " + strings.ReplaceAll(recordTitle, "|", "\\|") + " ")
|
||||
|
||||
if record.RawProperties != nil {
|
||||
var props map[string]interface{}
|
||||
json.Unmarshal(record.RawProperties, &props)
|
||||
for _, name := range propNames {
|
||||
val := ""
|
||||
if propMap, ok := props[name].(map[string]interface{}); ok {
|
||||
val = propertyToString(propMap)
|
||||
}
|
||||
val = strings.ReplaceAll(val, "\n", "<br>")
|
||||
val = strings.ReplaceAll(val, "|", "\\|")
|
||||
content.WriteString("| " + val + " ")
|
||||
}
|
||||
}
|
||||
content.WriteString("|\n")
|
||||
|
||||
// Fetch blocks for the record, in case it has page content
|
||||
blocks, err := client.GetBlockChildrenAll(ctx, record.ID)
|
||||
if err == nil && len(blocks) > 0 {
|
||||
resolveFileUploads(ctx, client, blocks)
|
||||
markdown, _ := BlocksToMarkdown(blocks)
|
||||
if strings.TrimSpace(markdown) != "" {
|
||||
extraContent.WriteString("\n## " + recordTitle + " 内容\n\n" + markdown + "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if extraContent.Len() > 0 {
|
||||
content.WriteString("\n" + extraContent.String())
|
||||
}
|
||||
|
||||
bodyStr := strings.TrimSpace(content.String())
|
||||
if bodyStr == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
updatedAt := time.Now()
|
||||
if len(records) > 0 {
|
||||
updatedAt = records[0].LastEditedTime
|
||||
}
|
||||
|
||||
return &types.FetchedItem{
|
||||
ExternalID: id,
|
||||
Title: title,
|
||||
Content: []byte(bodyStr),
|
||||
ContentType: contentTypeMarkdown,
|
||||
FileName: title + ".md",
|
||||
URL: fmt.Sprintf("https://notion.so/%s", strings.ReplaceAll(id, "-", "")),
|
||||
UpdatedAt: updatedAt,
|
||||
Metadata: map[string]string{
|
||||
"channel": types.ChannelNotion,
|
||||
"object_type": objectTypeDatabase,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// discoverAllResources uses the Search API to find all pages and data_sources,
|
||||
// then filters by parent chain via BFS from the selected resource IDs.
|
||||
// Returns the included pages plus the excluded set (visible pages NOT under any
|
||||
|
||||
@@ -2,6 +2,7 @@ package notion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -121,16 +122,13 @@ func TestConnectorFetchAll_Database(t *testing.T) {
|
||||
t.Fatal("expected at least 1 item")
|
||||
}
|
||||
|
||||
// Each database record should be synced as an individual knowledge item
|
||||
// The entire database should be synced as a single table knowledge item
|
||||
found := false
|
||||
for _, item := range items {
|
||||
if item.ExternalID == "record-1" {
|
||||
if item.ExternalID == "db-1" {
|
||||
found = true
|
||||
if item.Metadata["object_type"] != "page" {
|
||||
t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "page")
|
||||
}
|
||||
if item.Metadata["database"] != "Test Database" {
|
||||
t.Errorf("database = %q, want %q", item.Metadata["database"], "Test Database")
|
||||
if item.Metadata["object_type"] != "database" {
|
||||
t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "database")
|
||||
}
|
||||
if item.ContentType != "text/markdown" {
|
||||
t.Errorf("ContentType = %q", item.ContentType)
|
||||
@@ -138,10 +136,15 @@ func TestConnectorFetchAll_Database(t *testing.T) {
|
||||
if len(item.Content) == 0 {
|
||||
t.Error("expected non-empty content")
|
||||
}
|
||||
// Verify it contains table headers
|
||||
contentStr := string(item.Content)
|
||||
if !strings.Contains(contentStr, "| Title |") {
|
||||
t.Errorf("expected table header in content, got: %s", contentStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("record-1 not found in items")
|
||||
t.Error("db-1 not found in items")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestBlocksToMarkdown_Paragraph(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, attachments := BlocksToMarkdown(blocks)
|
||||
if md != "Hello world\n\n" {
|
||||
if md != "Hello world\n" {
|
||||
t.Errorf("markdown = %q", md)
|
||||
}
|
||||
if len(attachments) != 0 {
|
||||
@@ -161,7 +161,7 @@ func TestBlocksToMarkdown_Headings(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "# H1\n\n## H2\n\n### H3\n\n"
|
||||
expected := "# H1\n\n## H2\n\n### H3\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -177,7 +177,7 @@ func TestBlocksToMarkdown_Code(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "```go\nfmt.Println()\n```\n\n"
|
||||
expected := "```go\nfmt.Println()\n```\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -197,7 +197,7 @@ func TestBlocksToMarkdown_List(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "- Item 1\n- Item 2\n\n"
|
||||
expected := "- Item 1\n- Item 2\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -219,7 +219,7 @@ func TestBlocksToMarkdown_ToDo(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "- [ ] Unchecked\n- [x] Checked\n\n"
|
||||
expected := "- [ ] Unchecked\n- [x] Checked\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -234,7 +234,7 @@ func TestBlocksToMarkdown_Quote(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "> A wise quote\n\n"
|
||||
expected := "> A wise quote\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -245,7 +245,7 @@ func TestBlocksToMarkdown_Divider(t *testing.T) {
|
||||
{ID: "blk-div", Type: "divider"},
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
if md != "---\n\n" {
|
||||
if md != "---\n" {
|
||||
t.Errorf("markdown = %q", md)
|
||||
}
|
||||
}
|
||||
@@ -257,7 +257,7 @@ func TestBlocksToMarkdown_Equation(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
expected := "$$E = mc^2$$\n\n"
|
||||
expected := "$$E = mc^2$$\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -277,7 +277,7 @@ func TestBlocksToMarkdown_Image(t *testing.T) {
|
||||
}),
|
||||
}
|
||||
md, attachments := BlocksToMarkdown(blocks)
|
||||
if md != "\n\n" {
|
||||
if md != "\n" {
|
||||
t.Errorf("markdown = %q", md)
|
||||
}
|
||||
if len(attachments) != 1 {
|
||||
@@ -296,7 +296,7 @@ func TestBlocksToMarkdown_UnknownType(t *testing.T) {
|
||||
{ID: "blk-unknown", Type: "future_block_type"},
|
||||
}
|
||||
md, _ := BlocksToMarkdown(blocks)
|
||||
if md != "" {
|
||||
if md != "\n" {
|
||||
t.Errorf("unknown block should produce empty output, got %q", md)
|
||||
}
|
||||
}
|
||||
@@ -323,7 +323,7 @@ func TestBlocksToMarkdown_Table(t *testing.T) {
|
||||
},
|
||||
}
|
||||
md, _ := BlocksToMarkdown([]notionBlock{tableBlock})
|
||||
expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\n"
|
||||
expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
@@ -345,7 +345,7 @@ func TestBlocksToMarkdown_NestedList(t *testing.T) {
|
||||
}
|
||||
|
||||
md, _ := BlocksToMarkdown([]notionBlock{parent})
|
||||
expected := "- Parent\n - Child\n\n"
|
||||
expected := "- Parent\n - Child\n"
|
||||
if md != expected {
|
||||
t.Errorf("markdown = %q, want %q", md, expected)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user