fix: update environment configuration for debugging and logging, enhance SQLite FTS5 table management, and improve API base URL handling

This commit is contained in:
wizardchen
2026-04-11 05:48:53 +08:00
committed by lyingbug
parent 8a4d58726e
commit 9a2b54536d
35 changed files with 1486 additions and 341 deletions
+119 -27
View File
@@ -394,9 +394,6 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id
if err != nil || len(records) == 0 {
return nil
}
// Same database may be reached via two different IDs (data_source_id from search
// vs database_container_id from child_database blocks). Mark the canonical
// data_source_id as visited so the second path is deduped.
if queryID != "" && queryID != id {
if visited[queryID] {
return nil
@@ -404,18 +401,15 @@ func (c *Connector) fetchDatabase(ctx context.Context, client *notionClient, id
visited[queryID] = true
}
propNames := extractPropertySchema(records[0])
var items []types.FetchedItem
for _, record := range records {
if record.InTrash {
continue
}
item := c.buildRecordItem(ctx, client, record, propNames, dbTitle)
if item != nil {
items = append(items, *item)
}
visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls
}
return items
item := c.buildDatabaseItem(ctx, client, id, dbTitle, records)
if item != nil {
return []types.FetchedItem{*item}
}
return nil
}
// fetchDatabaseIncremental syncs only changed records by comparing edit times against cursor.
@@ -438,32 +432,33 @@ func (c *Connector) fetchDatabaseIncremental(ctx context.Context, client *notion
}
recordEditTimes := make(map[string]time.Time, len(records))
var propNames []string
if len(records) > 0 {
propNames = extractPropertySchema(records[0])
}
var items []types.FetchedItem
changedCount := 0
for _, record := range records {
visited[record.ID] = true // Mark records as visited to avoid duplicate fetchPage calls
if record.InTrash {
continue
}
recordEditTimes[record.ID] = record.LastEditedTime
prevTime, existed := prevEditTimes[record.ID]
if existed && record.LastEditedTime.Equal(prevTime) {
continue
}
changedCount++
item := c.buildRecordItem(ctx, client, record, propNames, dbTitle)
if item != nil {
items = append(items, *item)
if !existed || !record.LastEditedTime.Equal(prevTime) {
changedCount++
}
}
logger.Infof(ctx, "[Notion] database %s incremental: %d changed out of %d records", id, changedCount, len(records))
return items, recordEditTimes
// Rebuild the entire database table if any record changed, or if we don't have previous times (first sync)
if changedCount > 0 || len(prevEditTimes) == 0 {
item := c.buildDatabaseItem(ctx, client, id, dbTitle, records)
if item != nil {
return []types.FetchedItem{*item}, recordEditTimes
}
}
return nil, recordEditTimes
}
// queryDatabaseRecords resolves the database ID and queries all records.
@@ -545,6 +540,103 @@ func (c *Connector) buildRecordItem(ctx context.Context, client *notionClient, r
}
}
// buildDatabaseItem converts a database into a single Markdown table document.
func (c *Connector) buildDatabaseItem(ctx context.Context, client *notionClient, id string, dbTitle string, records []notionPage) *types.FetchedItem {
if len(records) == 0 {
return nil
}
propNames := extractPropertySchema(records[0])
var content strings.Builder
title := dbTitle
if title == "" {
title = defaultUntitledName
}
content.WriteString("# " + title + "\n\n")
// Table header
content.WriteString("| Title ")
for _, name := range propNames {
content.WriteString("| " + strings.ReplaceAll(name, "|", "\\|") + " ")
}
content.WriteString("|\n|")
content.WriteString("---|")
for range propNames {
content.WriteString("---|")
}
content.WriteString("\n")
var extraContent strings.Builder
// Table rows
for _, record := range records {
if record.InTrash {
continue
}
recordTitle := record.Title
if recordTitle == "" {
recordTitle = defaultUntitledName
}
content.WriteString("| " + strings.ReplaceAll(recordTitle, "|", "\\|") + " ")
if record.RawProperties != nil {
var props map[string]interface{}
json.Unmarshal(record.RawProperties, &props)
for _, name := range propNames {
val := ""
if propMap, ok := props[name].(map[string]interface{}); ok {
val = propertyToString(propMap)
}
val = strings.ReplaceAll(val, "\n", "<br>")
val = strings.ReplaceAll(val, "|", "\\|")
content.WriteString("| " + val + " ")
}
}
content.WriteString("|\n")
// Fetch blocks for the record, in case it has page content
blocks, err := client.GetBlockChildrenAll(ctx, record.ID)
if err == nil && len(blocks) > 0 {
resolveFileUploads(ctx, client, blocks)
markdown, _ := BlocksToMarkdown(blocks)
if strings.TrimSpace(markdown) != "" {
extraContent.WriteString("\n## " + recordTitle + " 内容\n\n" + markdown + "\n")
}
}
}
if extraContent.Len() > 0 {
content.WriteString("\n" + extraContent.String())
}
bodyStr := strings.TrimSpace(content.String())
if bodyStr == "" {
return nil
}
updatedAt := time.Now()
if len(records) > 0 {
updatedAt = records[0].LastEditedTime
}
return &types.FetchedItem{
ExternalID: id,
Title: title,
Content: []byte(bodyStr),
ContentType: contentTypeMarkdown,
FileName: title + ".md",
URL: fmt.Sprintf("https://notion.so/%s", strings.ReplaceAll(id, "-", "")),
UpdatedAt: updatedAt,
Metadata: map[string]string{
"channel": types.ChannelNotion,
"object_type": objectTypeDatabase,
},
}
}
// discoverAllResources uses the Search API to find all pages and data_sources,
// then filters by parent chain via BFS from the selected resource IDs.
// Returns the included pages plus the excluded set (visible pages NOT under any
@@ -2,6 +2,7 @@ package notion
import (
"context"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
@@ -121,16 +122,13 @@ func TestConnectorFetchAll_Database(t *testing.T) {
t.Fatal("expected at least 1 item")
}
// Each database record should be synced as an individual knowledge item
// The entire database should be synced as a single table knowledge item
found := false
for _, item := range items {
if item.ExternalID == "record-1" {
if item.ExternalID == "db-1" {
found = true
if item.Metadata["object_type"] != "page" {
t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "page")
}
if item.Metadata["database"] != "Test Database" {
t.Errorf("database = %q, want %q", item.Metadata["database"], "Test Database")
if item.Metadata["object_type"] != "database" {
t.Errorf("object_type = %q, want %q", item.Metadata["object_type"], "database")
}
if item.ContentType != "text/markdown" {
t.Errorf("ContentType = %q", item.ContentType)
@@ -138,10 +136,15 @@ func TestConnectorFetchAll_Database(t *testing.T) {
if len(item.Content) == 0 {
t.Error("expected non-empty content")
}
// Verify it contains table headers
contentStr := string(item.Content)
if !strings.Contains(contentStr, "| Title |") {
t.Errorf("expected table header in content, got: %s", contentStr)
}
}
}
if !found {
t.Error("record-1 not found in items")
t.Error("db-1 not found in items")
}
}
@@ -134,7 +134,7 @@ func TestBlocksToMarkdown_Paragraph(t *testing.T) {
}),
}
md, attachments := BlocksToMarkdown(blocks)
if md != "Hello world\n\n" {
if md != "Hello world\n" {
t.Errorf("markdown = %q", md)
}
if len(attachments) != 0 {
@@ -161,7 +161,7 @@ func TestBlocksToMarkdown_Headings(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "# H1\n\n## H2\n\n### H3\n\n"
expected := "# H1\n\n## H2\n\n### H3\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -177,7 +177,7 @@ func TestBlocksToMarkdown_Code(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "```go\nfmt.Println()\n```\n\n"
expected := "```go\nfmt.Println()\n```\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -197,7 +197,7 @@ func TestBlocksToMarkdown_List(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "- Item 1\n- Item 2\n\n"
expected := "- Item 1\n- Item 2\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -219,7 +219,7 @@ func TestBlocksToMarkdown_ToDo(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "- [ ] Unchecked\n- [x] Checked\n\n"
expected := "- [ ] Unchecked\n- [x] Checked\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -234,7 +234,7 @@ func TestBlocksToMarkdown_Quote(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "> A wise quote\n\n"
expected := "> A wise quote\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -245,7 +245,7 @@ func TestBlocksToMarkdown_Divider(t *testing.T) {
{ID: "blk-div", Type: "divider"},
}
md, _ := BlocksToMarkdown(blocks)
if md != "---\n\n" {
if md != "---\n" {
t.Errorf("markdown = %q", md)
}
}
@@ -257,7 +257,7 @@ func TestBlocksToMarkdown_Equation(t *testing.T) {
}),
}
md, _ := BlocksToMarkdown(blocks)
expected := "$$E = mc^2$$\n\n"
expected := "$$E = mc^2$$\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -277,7 +277,7 @@ func TestBlocksToMarkdown_Image(t *testing.T) {
}),
}
md, attachments := BlocksToMarkdown(blocks)
if md != "![A photo](https://s3.example.com/img.png)\n\n" {
if md != "![A photo](https://s3.example.com/img.png)\n" {
t.Errorf("markdown = %q", md)
}
if len(attachments) != 1 {
@@ -296,7 +296,7 @@ func TestBlocksToMarkdown_UnknownType(t *testing.T) {
{ID: "blk-unknown", Type: "future_block_type"},
}
md, _ := BlocksToMarkdown(blocks)
if md != "" {
if md != "\n" {
t.Errorf("unknown block should produce empty output, got %q", md)
}
}
@@ -323,7 +323,7 @@ func TestBlocksToMarkdown_Table(t *testing.T) {
},
}
md, _ := BlocksToMarkdown([]notionBlock{tableBlock})
expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\n"
expected := "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}
@@ -345,7 +345,7 @@ func TestBlocksToMarkdown_NestedList(t *testing.T) {
}
md, _ := BlocksToMarkdown([]notionBlock{parent})
expected := "- Parent\n - Child\n\n"
expected := "- Parent\n - Child\n"
if md != expected {
t.Errorf("markdown = %q, want %q", md, expected)
}