fix(schema): expand knowledge source length

internal/application/repository/knowledge_finalize_test.go

@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS knowledges (
     type VARCHAR(50) NOT NULL DEFAULT '',
     title VARCHAR(255) NOT NULL DEFAULT '',
     description TEXT,
-    source VARCHAR(128) NOT NULL DEFAULT '',
+    source VARCHAR(2048) NOT NULL DEFAULT '',
     parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
     enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
     embedding_model_id VARCHAR(64),

internal/application/repository/knowledge_source_schema_test.go

@@ -0,0 +1,67 @@
+package repository
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/types"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func TestKnowledgeSourceSchemaAllowsObjectStorageURLs(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open("file:knowledge_source_schema?mode=memory&cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("open sqlite: %v", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("get sqlite handle: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1)
+	if err := db.AutoMigrate(&types.Knowledge{}); err != nil {
+		t.Fatalf("auto migrate knowledge: %v", err)
+	}
+
+	var sourceType string
+	rows, err := db.Raw("PRAGMA table_info(knowledges)").Rows()
+	if err != nil {
+		t.Fatalf("inspect knowledge schema: %v", err)
+	}
+
+	for rows.Next() {
+		var cid int
+		var name, typ string
+		var notNull int
+		var defaultValue any
+		var pk int
+		if err := rows.Scan(&cid, &name, &typ, &notNull, &defaultValue, &pk); err != nil {
+			t.Fatalf("scan column info: %v", err)
+		}
+		if name == "source" {
+			sourceType = strings.ToLower(typ)
+			break
+		}
+	}
+	if err := rows.Close(); err != nil {
+		t.Fatalf("close column info rows: %v", err)
+	}
+	if sourceType != "varchar(2048)" {
+		t.Fatalf("knowledge source column type = %q, want varchar(2048)", sourceType)
+	}
+
+	longURL := "https://example-bucket.cos.ap-beijing.myqcloud.com/test/" +
+		strings.Repeat("encoded-path-segment-", 20) + ".docx"
+	knowledge := &types.Knowledge{
+		TenantID:        1,
+		KnowledgeBaseID: "kb-1",
+		Type:            types.KnowledgeTypeManual,
+		Title:           "long source",
+		Source:          longURL,
+		ParseStatus:     types.ParseStatusPending,
+		EnableStatus:    "enabled",
+	}
+	if err := db.Create(knowledge).Error; err != nil {
+		t.Fatalf("create knowledge with long source: %v", err)
+	}
+}

internal/types/knowledge.go

@@ -124,7 +124,7 @@ type Knowledge struct {
 	// Description of the knowledge
 	Description string `json:"description"`
 	// Source of the knowledge (e.g. URL address for url type, "manual" for manual type)
-	Source string `json:"source"`
+	Source string `json:"source"             gorm:"type:varchar(2048)"`
 	// Channel indicates through which channel the knowledge was ingested (web, api, browser_extension, wechat, etc.)
 	Channel string `json:"channel"            gorm:"type:varchar(50);default:'web'"`
 	// Parse status of the knowledge

migrations/mysql/00-init-db.sql

@@ -67,7 +67,7 @@ CREATE TABLE knowledges (
     type VARCHAR(50) NOT NULL,
     title VARCHAR(255) NOT NULL,
     description TEXT,
-    source VARCHAR(128) NOT NULL,
+    source VARCHAR(2048) NOT NULL,
     parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
     enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
     embedding_model_id VARCHAR(64),

migrations/paradedb/00-init-db.sql

@@ -82,7 +82,7 @@ CREATE TABLE IF NOT EXISTS knowledges (
     type VARCHAR(50) NOT NULL,
     title VARCHAR(255) NOT NULL,
     description TEXT,
-    source VARCHAR(128) NOT NULL,
+    source VARCHAR(2048) NOT NULL,
     parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
     enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
     embedding_model_id VARCHAR(64),

migrations/sqlite/000000_init.up.sql

@@ -90,7 +90,7 @@ CREATE TABLE IF NOT EXISTS knowledges (
     type VARCHAR(50) NOT NULL,
     title VARCHAR(255) NOT NULL,
     description TEXT,
-    source VARCHAR(128) NOT NULL,
+    source VARCHAR(2048) NOT NULL,
     parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
     enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
     embedding_model_id VARCHAR(64),

migrations/versioned/000000_init.up.sql

@@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS knowledges (
     type VARCHAR(50) NOT NULL,
     title VARCHAR(255) NOT NULL,
     description TEXT,
-    source VARCHAR(128) NOT NULL,
+    source VARCHAR(2048) NOT NULL,
     parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed',
     enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled',
     embedding_model_id VARCHAR(64),

migrations/versioned/000058_expand_knowledge_source.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE knowledges
+    ALTER COLUMN source TYPE VARCHAR(128);

migrations/versioned/000058_expand_knowledge_source.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE knowledges
+    ALTER COLUMN source TYPE VARCHAR(2048);