diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.go index 830fef4c..176ad466 100644 --- a/internal/application/repository/knowledge_finalize_test.go +++ b/internal/application/repository/knowledge_finalize_test.go @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS knowledges ( type VARCHAR(50) NOT NULL DEFAULT '', title VARCHAR(255) NOT NULL DEFAULT '', description TEXT, - source VARCHAR(128) NOT NULL DEFAULT '', + source VARCHAR(2048) NOT NULL DEFAULT '', parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), diff --git a/internal/application/repository/knowledge_source_schema_test.go b/internal/application/repository/knowledge_source_schema_test.go new file mode 100644 index 00000000..6c380982 --- /dev/null +++ b/internal/application/repository/knowledge_source_schema_test.go @@ -0,0 +1,67 @@ +package repository + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestKnowledgeSourceSchemaAllowsObjectStorageURLs(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:knowledge_source_schema?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("get sqlite handle: %v", err) + } + sqlDB.SetMaxOpenConns(1) + if err := db.AutoMigrate(&types.Knowledge{}); err != nil { + t.Fatalf("auto migrate knowledge: %v", err) + } + + var sourceType string + rows, err := db.Raw("PRAGMA table_info(knowledges)").Rows() + if err != nil { + t.Fatalf("inspect knowledge schema: %v", err) + } + + for rows.Next() { + var cid int + var name, typ string + var notNull int + var defaultValue any + var pk int + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { + t.Fatalf("scan column info: %v", err) + } + if name == "source" { + sourceType = strings.ToLower(typ) + break + } + } + if err := rows.Close(); err != nil { + t.Fatalf("close column info rows: %v", err) + } + if sourceType != "varchar(2048)" { + t.Fatalf("knowledge source column type = %q, want varchar(2048)", sourceType) + } + + longURL := "https://example-bucket.cos.ap-beijing.myqcloud.com/test/" + + strings.Repeat("encoded-path-segment-", 20) + ".docx" + knowledge := &types.Knowledge{ + TenantID: 1, + KnowledgeBaseID: "kb-1", + Type: types.KnowledgeTypeManual, + Title: "long source", + Source: longURL, + ParseStatus: types.ParseStatusPending, + EnableStatus: "enabled", + } + if err := db.Create(knowledge).Error; err != nil { + t.Fatalf("create knowledge with long source: %v", err) + } +} diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index d5867f1b..439c6a9a 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -124,7 +124,7 @@ type Knowledge struct { // Description of the knowledge Description string `json:"description"` // Source of the knowledge (e.g. URL address for url type, "manual" for manual type) - Source string `json:"source"` + Source string `json:"source" gorm:"type:varchar(2048)"` // Channel indicates through which channel the knowledge was ingested (web, api, browser_extension, wechat, etc.) Channel string `json:"channel" gorm:"type:varchar(50);default:'web'"` // Parse status of the knowledge diff --git a/migrations/mysql/00-init-db.sql b/migrations/mysql/00-init-db.sql index 319f6102..d6634bd0 100644 --- a/migrations/mysql/00-init-db.sql +++ b/migrations/mysql/00-init-db.sql @@ -67,7 +67,7 @@ CREATE TABLE knowledges ( type VARCHAR(50) NOT NULL, title VARCHAR(255) NOT NULL, description TEXT, - source VARCHAR(128) NOT NULL, + source VARCHAR(2048) NOT NULL, parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), diff --git a/migrations/paradedb/00-init-db.sql b/migrations/paradedb/00-init-db.sql index 6bc78389..7df315cd 100644 --- a/migrations/paradedb/00-init-db.sql +++ b/migrations/paradedb/00-init-db.sql @@ -82,7 +82,7 @@ CREATE TABLE IF NOT EXISTS knowledges ( type VARCHAR(50) NOT NULL, title VARCHAR(255) NOT NULL, description TEXT, - source VARCHAR(128) NOT NULL, + source VARCHAR(2048) NOT NULL, parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index d95a8194..f06b4f91 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -90,7 +90,7 @@ CREATE TABLE IF NOT EXISTS knowledges ( type VARCHAR(50) NOT NULL, title VARCHAR(255) NOT NULL, description TEXT, - source VARCHAR(128) NOT NULL, + source VARCHAR(2048) NOT NULL, parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), diff --git a/migrations/versioned/000000_init.up.sql b/migrations/versioned/000000_init.up.sql index 8a5b9988..57cbbbbe 100644 --- a/migrations/versioned/000000_init.up.sql +++ b/migrations/versioned/000000_init.up.sql @@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS knowledges ( type VARCHAR(50) NOT NULL, title VARCHAR(255) NOT NULL, description TEXT, - source VARCHAR(128) NOT NULL, + source VARCHAR(2048) NOT NULL, parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', embedding_model_id VARCHAR(64), diff --git a/migrations/versioned/000058_expand_knowledge_source.down.sql b/migrations/versioned/000058_expand_knowledge_source.down.sql new file mode 100644 index 00000000..81ceca27 --- /dev/null +++ b/migrations/versioned/000058_expand_knowledge_source.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE knowledges + ALTER COLUMN source TYPE VARCHAR(128); diff --git a/migrations/versioned/000058_expand_knowledge_source.up.sql b/migrations/versioned/000058_expand_knowledge_source.up.sql new file mode 100644 index 00000000..e56d865c --- /dev/null +++ b/migrations/versioned/000058_expand_knowledge_source.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE knowledges + ALTER COLUMN source TYPE VARCHAR(2048);