From 127654ee786806a0dbb8032c0c38c7eb3adfe82d Mon Sep 17 00:00:00 2001 From: "ochan.kwon" Date: Mon, 20 Apr 2026 11:18:51 +0900 Subject: [PATCH] feat: add vector_store_id column to knowledge_bases for per-KB vector store binding --- .../repository/knowledgebase_sqlite_test.go | 217 ++++++++++++++++++ internal/types/knowledgebase.go | 7 + internal/types/knowledgebase_test.go | 105 ++++++++- migrations/sqlite/000000_init.up.sql | 3 + .../000036_kb_vector_store_id.down.sql | 30 +++ .../000036_kb_vector_store_id.up.sql | 39 ++++ 6 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 internal/application/repository/knowledgebase_sqlite_test.go create mode 100644 migrations/versioned/000036_kb_vector_store_id.down.sql create mode 100644 migrations/versioned/000036_kb_vector_store_id.up.sql diff --git a/internal/application/repository/knowledgebase_sqlite_test.go b/internal/application/repository/knowledgebase_sqlite_test.go new file mode 100644 index 00000000..dc55c47f --- /dev/null +++ b/internal/application/repository/knowledgebase_sqlite_test.go @@ -0,0 +1,217 @@ +package repository + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// knowledgeBasesTestDDL mirrors the `knowledge_bases` section of +// migrations/sqlite/000000_init.up.sql. We inline the DDL here instead of +// using GORM AutoMigrate because KnowledgeBase carries fields tagged with +// `type:jsonb`, which AutoMigrate does not map cleanly onto SQLite. +const knowledgeBasesTestDDL = ` +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + tenant_id INTEGER NOT NULL, + type VARCHAR(32) NOT NULL DEFAULT 'document', + chunking_config TEXT NOT NULL DEFAULT '{}', + image_processing_config TEXT NOT NULL DEFAULT '{}', + embedding_model_id VARCHAR(64) NOT NULL, + summary_model_id VARCHAR(64) NOT NULL, + cos_config TEXT NOT NULL DEFAULT '{}', + storage_provider_config TEXT DEFAULT NULL, + vlm_config TEXT NOT NULL DEFAULT '{}', + extract_config TEXT NULL DEFAULT NULL, + faq_config TEXT, + question_generation_config TEXT NULL, + is_temporary BOOLEAN NOT NULL DEFAULT 0, + is_pinned INTEGER NOT NULL DEFAULT 0, + pinned_at DATETIME NULL, + asr_config TEXT, + vector_store_id VARCHAR(36), + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + deleted_at DATETIME +); +` + +// setupKBTestDB creates an in-memory SQLite database containing the +// knowledge_bases table with the vector_store_id column included, so that +// tests can exercise the GORM behavior of the VectorStoreID field. +func setupKBTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.Exec(knowledgeBasesTestDDL).Error) + return db +} + +// makeKB builds a minimal KnowledgeBase suitable for insert. Only the columns +// required by the schema above and any field we explicitly care about are set; +// the rest are left at their Go zero values and serialize to "{}" via their +// custom JSON valuers. +func makeKB(vectorStoreID *string) *types.KnowledgeBase { + return &types.KnowledgeBase{ + ID: uuid.New().String(), + Name: "test-kb", + TenantID: 1, + EmbeddingModelID: "test-embed", + SummaryModelID: "test-summary", + VectorStoreID: vectorStoreID, + } +} + +func kbStrPtr(s string) *string { return &s } + +// reloadKB fetches the KnowledgeBase back from the database using its ID. +func reloadKB(t *testing.T, db *gorm.DB, id string) types.KnowledgeBase { + t.Helper() + var got types.KnowledgeBase + require.NoError(t, db.First(&got, "id = ?", id).Error) + return got +} + +// TestKnowledgeBase_VectorStoreID_Save_Immutable verifies that db.Save(kb) +// — which WeKnora's UpdateKnowledgeBase repository path uses today — does +// NOT overwrite the vector_store_id column, thanks to the GORM +// `<-:create` tag. This is the primary automated defense for the H1 review +// finding (silent overwrite of the binding via full-struct UPDATE). +func TestKnowledgeBase_VectorStoreID_Save_Immutable(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(kbStrPtr("store-A")) + require.NoError(t, db.Create(original).Error) + + // Attempt to overwrite through the canonical Save() path. + mutated := *original + mutated.Name = "updated-name" + mutated.VectorStoreID = kbStrPtr("store-B") + require.NoError(t, db.Save(&mutated).Error) + + reloaded := reloadKB(t, db, original.ID) + + // Non-immutable fields must update. + assert.Equal(t, "updated-name", reloaded.Name, "Name must update through Save") + + // Immutable field must NOT update. + require.NotNil(t, reloaded.VectorStoreID, "VectorStoreID must remain set to the original value") + assert.Equal(t, "store-A", *reloaded.VectorStoreID, + "db.Save must not overwrite vector_store_id (enforced by GORM `<-:create` tag)") +} + +// TestKnowledgeBase_VectorStoreID_Updates_Immutable verifies that +// db.Updates(struct{...}) also honors `<-:create` and silently ignores the +// vector_store_id column. +func TestKnowledgeBase_VectorStoreID_Updates_Immutable(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(kbStrPtr("store-A")) + require.NoError(t, db.Create(original).Error) + + // Partial struct update that includes vector_store_id must not take effect. + err := db.Model(&types.KnowledgeBase{}). + Where("id = ?", original.ID). + Updates(types.KnowledgeBase{ + Name: "updated-name", + VectorStoreID: kbStrPtr("store-B"), + }).Error + require.NoError(t, err) + + reloaded := reloadKB(t, db, original.ID) + + assert.Equal(t, "updated-name", reloaded.Name) + require.NotNil(t, reloaded.VectorStoreID) + assert.Equal(t, "store-A", *reloaded.VectorStoreID, + "db.Updates must not overwrite vector_store_id (enforced by GORM `<-:create` tag)") +} + +// TestKnowledgeBase_VectorStoreID_Select_StillImmutable verifies that even +// an explicit Select("vector_store_id").Updates(...) does NOT bypass the +// `<-:create` tag. GORM's `<-:create` is strict: it withholds write +// permission from every UPDATE path, including selective updates. This is +// a stronger guarantee than a typical "column is protected unless you +// opt in" scheme. +// +// The implication for Phase 4 (cross-store migration) is that ORM-level +// updates cannot move bindings. Migration code must use raw SQL (db.Exec) +// — see TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch below. +func TestKnowledgeBase_VectorStoreID_Select_StillImmutable(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(kbStrPtr("store-A")) + require.NoError(t, db.Create(original).Error) + + err := db.Model(&types.KnowledgeBase{}). + Where("id = ?", original.ID). + Select("vector_store_id"). + Updates(map[string]interface{}{"vector_store_id": "store-B"}).Error + require.NoError(t, err) + + reloaded := reloadKB(t, db, original.ID) + require.NotNil(t, reloaded.VectorStoreID) + assert.Equal(t, "store-A", *reloaded.VectorStoreID, + "Select(\"vector_store_id\").Updates must NOT bypass <-:create; ORM is fully locked") +} + +// TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch documents that raw +// SQL UPDATE bypasses the GORM tag system entirely, as expected. Phase 4 +// cross-store migration is the only sanctioned caller of this path, and +// it must do so through a dedicated, narrowly-scoped repository method +// rather than through general Save/Updates/Select flows. +func TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(kbStrPtr("store-A")) + require.NoError(t, db.Create(original).Error) + + require.NoError(t, + db.Exec("UPDATE knowledge_bases SET vector_store_id = ? WHERE id = ?", + "store-B", original.ID).Error) + + reloaded := reloadKB(t, db, original.ID) + require.NotNil(t, reloaded.VectorStoreID) + assert.Equal(t, "store-B", *reloaded.VectorStoreID, + "raw SQL UPDATE bypasses GORM tags; Phase 4 migration must use this path through a dedicated method") +} + +// TestKnowledgeBase_VectorStoreID_Roundtrip_Nil verifies that a nil +// VectorStoreID is persisted as SQL NULL and loaded back as nil. +func TestKnowledgeBase_VectorStoreID_Roundtrip_Nil(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(nil) + require.NoError(t, db.Create(original).Error) + + reloaded := reloadKB(t, db, original.ID) + assert.Nil(t, reloaded.VectorStoreID, + "nil VectorStoreID must round-trip through the database as nil") + + // Extra assurance: column is actually NULL, not an empty string. + var nullCount int64 + require.NoError(t, + db.Table("knowledge_bases"). + Where("id = ? AND vector_store_id IS NULL", original.ID). + Count(&nullCount).Error) + assert.Equal(t, int64(1), nullCount, "nil must be stored as SQL NULL") +} + +// TestKnowledgeBase_VectorStoreID_Roundtrip_Value verifies that a non-nil +// VectorStoreID is persisted and loaded back with the same value. +func TestKnowledgeBase_VectorStoreID_Roundtrip_Value(t *testing.T) { + db := setupKBTestDB(t) + + original := makeKB(kbStrPtr("store-uuid-42")) + require.NoError(t, db.Create(original).Error) + + reloaded := reloadKB(t, db, original.ID) + require.NotNil(t, reloaded.VectorStoreID) + assert.Equal(t, "store-uuid-42", *reloaded.VectorStoreID) +} diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 462e5829..48100733 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -66,6 +66,13 @@ type KnowledgeBase struct { StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"` // Deprecated: legacy COS config column. Kept for backward compatibility with old data. StorageConfig StorageConfig `yaml:"-" json:"storage_config" gorm:"column:cos_config;type:json"` + // VectorStoreID references the VectorStore this knowledge base is bound to. + // When nil, the KB falls back to the tenant's effective engines derived from + // the RETRIEVE_DRIVER environment variable (env store flow). + // This field is set once at creation time and must not be modified afterwards; + // enforcement lives at the GORM layer (`<-:create`) plus the service-layer + // KB update path, which omits this field from its update DTO. + VectorStoreID *string `yaml:"vector_store_id" json:"vector_store_id,omitempty" gorm:"column:vector_store_id;type:varchar(36);<-:create"` // Extract config ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"` // FAQConfig stores FAQ specific configuration such as indexing strategy diff --git a/internal/types/knowledgebase_test.go b/internal/types/knowledgebase_test.go index 2f860b9f..a53cba2c 100644 --- a/internal/types/knowledgebase_test.go +++ b/internal/types/knowledgebase_test.go @@ -1,6 +1,10 @@ package types -import "testing" +import ( + "encoding/json" + "strings" + "testing" +) func TestParseProviderScheme(t *testing.T) { tests := []struct { @@ -53,3 +57,102 @@ func TestInferStorageFromFilePath(t *testing.T) { }) } } + +// strPtr returns a pointer to the given string, used to express *string literals in tests. +func strPtr(s string) *string { return &s } + +// TestKnowledgeBase_VectorStoreID_JSON covers (un)marshaling of the nullable +// VectorStoreID pointer, including the `omitempty` behavior and the three +// distinct JSON inputs (missing field, explicit null, explicit value). +// +// The `empty string pointer` case is fixed here as a behavior snapshot: PR1 +// accepts it and stores &"" verbatim because validation has not been added +// yet. A follow-up PR that introduces validation will convert this case into +// a rejection and flip the expectation in this test accordingly. +func TestKnowledgeBase_VectorStoreID_JSON(t *testing.T) { + t.Run("marshal nil omits field (omitempty)", func(t *testing.T) { + kb := KnowledgeBase{ID: "kb-1", VectorStoreID: nil} + b, err := json.Marshal(kb) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } + if strings.Contains(string(b), `"vector_store_id"`) { + t.Errorf("expected vector_store_id to be omitted when nil, got: %s", b) + } + }) + + t.Run("marshal non-nil includes value", func(t *testing.T) { + kb := KnowledgeBase{ID: "kb-1", VectorStoreID: strPtr("store-uuid")} + b, err := json.Marshal(kb) + if err != nil { + t.Fatalf("json.Marshal returned error: %v", err) + } + if !strings.Contains(string(b), `"vector_store_id":"store-uuid"`) { + t.Errorf("expected vector_store_id to be serialized, got: %s", b) + } + }) + + unmarshalCases := []struct { + name string + body string + wantNil bool + wantValue string + }{ + {name: "missing field", body: `{"id":"kb-1"}`, wantNil: true}, + {name: "explicit null", body: `{"id":"kb-1","vector_store_id":null}`, wantNil: true}, + {name: "explicit value", body: `{"id":"kb-1","vector_store_id":"store-uuid"}`, wantValue: "store-uuid"}, + {name: "empty string pointer", body: `{"id":"kb-1","vector_store_id":""}`, wantValue: ""}, + } + for _, tt := range unmarshalCases { + t.Run(tt.name, func(t *testing.T) { + var kb KnowledgeBase + if err := json.Unmarshal([]byte(tt.body), &kb); err != nil { + t.Fatalf("json.Unmarshal returned error: %v", err) + } + if tt.wantNil { + if kb.VectorStoreID != nil { + t.Errorf("expected VectorStoreID to be nil, got &%q", *kb.VectorStoreID) + } + return + } + if kb.VectorStoreID == nil { + t.Fatalf("expected VectorStoreID to be &%q, got nil", tt.wantValue) + } + if *kb.VectorStoreID != tt.wantValue { + t.Errorf("expected VectorStoreID = %q, got %q", tt.wantValue, *kb.VectorStoreID) + } + }) + } +} + +// TestKnowledgeBase_UnmarshalJSON_WithVectorStoreID verifies that the custom +// UnmarshalJSON on KnowledgeBase (which shadows cos_config for legacy +// compatibility) still delegates the new vector_store_id field to the alias +// type path, without interfering with StorageProviderConfig inference. +func TestKnowledgeBase_UnmarshalJSON_WithVectorStoreID(t *testing.T) { + // Legacy cos_config + new vector_store_id in the same payload: both must map correctly. + body := `{ + "id": "kb-1", + "cos_config": {"provider": "cos", "bucket_name": "legacy-bucket"}, + "vector_store_id": "store-uuid" + }` + + var kb KnowledgeBase + if err := json.Unmarshal([]byte(body), &kb); err != nil { + t.Fatalf("json.Unmarshal returned error: %v", err) + } + + if kb.VectorStoreID == nil || *kb.VectorStoreID != "store-uuid" { + t.Errorf("expected VectorStoreID = &\"store-uuid\", got %v", kb.VectorStoreID) + } + if kb.StorageConfig.Provider != "cos" { + t.Errorf("expected legacy StorageConfig.Provider = cos, got %q", kb.StorageConfig.Provider) + } + if kb.StorageProviderConfig == nil || kb.StorageProviderConfig.Provider != "cos" { + t.Errorf("expected StorageProviderConfig.Provider auto-populated from cos_config, got %v", kb.StorageProviderConfig) + } + + // Regression guard: the aux struct inside UnmarshalJSON must not shadow vector_store_id. + // If a future change introduces such a shadow, the value above would fail to populate. +} + diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index 4f7cb407..56499e0e 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -67,12 +67,15 @@ CREATE TABLE IF NOT EXISTS knowledge_bases ( is_pinned INTEGER NOT NULL DEFAULT 0, pinned_at DATETIME NULL, asr_config TEXT, + vector_store_id VARCHAR(36), created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, deleted_at DATETIME ); CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store + ON knowledge_bases(tenant_id, vector_store_id); CREATE TABLE IF NOT EXISTS knowledges ( id VARCHAR(36) PRIMARY KEY, diff --git a/migrations/versioned/000036_kb_vector_store_id.down.sql b/migrations/versioned/000036_kb_vector_store_id.down.sql new file mode 100644 index 00000000..003b9ace --- /dev/null +++ b/migrations/versioned/000036_kb_vector_store_id.down.sql @@ -0,0 +1,30 @@ +-- Migration: 000036_kb_vector_store_id (rollback) +-- Description: Remove vector_store_id column and its index from knowledge_bases. +-- +-- WARNING: This rollback is destructive. It permanently drops the vector_store_id column +-- and all KB ↔ VectorStore bindings. To preserve bindings before rollback: +-- +-- CREATE TABLE knowledge_bases_vsid_backup AS +-- SELECT id, tenant_id, vector_store_id FROM knowledge_bases +-- WHERE vector_store_id IS NOT NULL; +-- +-- This down migration does NOT block execution when non-NULL rows exist, because blocking +-- during an emergency rollback can make an incident worse. The RAISE NOTICE below logs the +-- number of rows whose bindings will be lost so operators can reconstruct later if needed. +DO $$ +DECLARE non_null_count BIGINT; +BEGIN + SELECT COUNT(*) INTO non_null_count + FROM knowledge_bases WHERE vector_store_id IS NOT NULL; + IF non_null_count > 0 THEN + RAISE NOTICE '[Migration 000036 DOWN] Dropping vector_store_id with % non-NULL rows; these bindings will be lost', non_null_count; + ELSE + RAISE NOTICE '[Migration 000036 DOWN] Removing vector_store_id column from knowledge_bases'; + END IF; +END $$; + +DROP INDEX IF EXISTS idx_knowledge_bases_tenant_vector_store; + +ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS vector_store_id; + +DO $$ BEGIN RAISE NOTICE '[Migration 000036 DOWN] vector_store_id column removed successfully'; END $$; diff --git a/migrations/versioned/000036_kb_vector_store_id.up.sql b/migrations/versioned/000036_kb_vector_store_id.up.sql new file mode 100644 index 00000000..2a3485f6 --- /dev/null +++ b/migrations/versioned/000036_kb_vector_store_id.up.sql @@ -0,0 +1,39 @@ +-- Migration: 000036_kb_vector_store_id +-- Description: Add vector_store_id column to knowledge_bases for per-KB VectorStore binding. +-- NULL means "use tenant's effective engines" (env store derived from RETRIEVE_DRIVER), +-- which preserves existing behavior for all pre-existing rows. +-- No foreign key: env store IDs are virtual (not present in vector_stores), and +-- VectorStore soft-delete must not cascade into KB references. Referential integrity is +-- enforced at the application layer. +-- On PostgreSQL 11+ this ALTER runs as metadata-only (O(1), no row rewrite) because the +-- column is nullable with no DEFAULT. PostgreSQL 10 and earlier are not supported. +DO $$ BEGIN RAISE NOTICE '[Migration 000036] Adding vector_store_id column to knowledge_bases'; END $$; + +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS vector_store_id VARCHAR(36); + +COMMENT ON COLUMN knowledge_bases.vector_store_id IS + 'References vector_stores.id. NULL means tenant default (env store derived from RETRIEVE_DRIVER). ' + 'Immutable after creation (enforced at ORM and service layer). No FK by design.'; + +-- Composite index: tenant_id first to align with the dominant query pattern. All KB queries +-- start with `WHERE tenant_id = $1`, so the composite index enforces tenant-scoped plans and +-- hardens against accidental cross-tenant access. A full (non-partial) index is used because +-- the target end-state has most KBs bound to a store, making a partial index short-lived. +-- Storage cost is negligible at typical KB row counts. +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store + ON knowledge_bases(tenant_id, vector_store_id); + +-- Note: CREATE INDEX here holds a SHARE lock briefly. For very large knowledge_bases tables +-- (hundreds of thousands of rows or more), operators may prefer CREATE INDEX CONCURRENTLY. +-- This requires running the statement outside a transaction, which the standard migration +-- runner does not support; execute the CONCURRENTLY variant from a separate operational +-- script in that case. Simply replacing the statement below with CONCURRENTLY inside this +-- file will cause the migration to fail and leave schema_migrations in a dirty state. + +DO $$ +DECLARE non_null_count BIGINT; +BEGIN + SELECT COUNT(*) INTO non_null_count + FROM knowledge_bases WHERE vector_store_id IS NOT NULL; + RAISE NOTICE '[Migration 000036] knowledge_bases.vector_store_id added (% non-NULL rows)', non_null_count; +END $$;