feat: add vector_store_id column to knowledge_bases for per-KB vector store binding

This commit is contained in:
ochan.kwon
2026-04-20 11:18:51 +09:00
committed by lyingbug
parent e3b493e316
commit 127654ee78
6 changed files with 400 additions and 1 deletions
@@ -0,0 +1,217 @@
package repository
import (
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// knowledgeBasesTestDDL mirrors the `knowledge_bases` section of
// migrations/sqlite/000000_init.up.sql. We inline the DDL here instead of
// using GORM AutoMigrate because KnowledgeBase carries fields tagged with
// `type:jsonb`, which AutoMigrate does not map cleanly onto SQLite.
const knowledgeBasesTestDDL = `
CREATE TABLE IF NOT EXISTS knowledge_bases (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
tenant_id INTEGER NOT NULL,
type VARCHAR(32) NOT NULL DEFAULT 'document',
chunking_config TEXT NOT NULL DEFAULT '{}',
image_processing_config TEXT NOT NULL DEFAULT '{}',
embedding_model_id VARCHAR(64) NOT NULL,
summary_model_id VARCHAR(64) NOT NULL,
cos_config TEXT NOT NULL DEFAULT '{}',
storage_provider_config TEXT DEFAULT NULL,
vlm_config TEXT NOT NULL DEFAULT '{}',
extract_config TEXT NULL DEFAULT NULL,
faq_config TEXT,
question_generation_config TEXT NULL,
is_temporary BOOLEAN NOT NULL DEFAULT 0,
is_pinned INTEGER NOT NULL DEFAULT 0,
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
`
// setupKBTestDB creates an in-memory SQLite database containing the
// knowledge_bases table with the vector_store_id column included, so that
// tests can exercise the GORM behavior of the VectorStoreID field.
func setupKBTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(knowledgeBasesTestDDL).Error)
return db
}
// makeKB builds a minimal KnowledgeBase suitable for insert. Only the columns
// required by the schema above and any field we explicitly care about are set;
// the rest are left at their Go zero values and serialize to "{}" via their
// custom JSON valuers.
func makeKB(vectorStoreID *string) *types.KnowledgeBase {
return &types.KnowledgeBase{
ID: uuid.New().String(),
Name: "test-kb",
TenantID: 1,
EmbeddingModelID: "test-embed",
SummaryModelID: "test-summary",
VectorStoreID: vectorStoreID,
}
}
func kbStrPtr(s string) *string { return &s }
// reloadKB fetches the KnowledgeBase back from the database using its ID.
func reloadKB(t *testing.T, db *gorm.DB, id string) types.KnowledgeBase {
t.Helper()
var got types.KnowledgeBase
require.NoError(t, db.First(&got, "id = ?", id).Error)
return got
}
// TestKnowledgeBase_VectorStoreID_Save_Immutable verifies that db.Save(kb)
// — which WeKnora's UpdateKnowledgeBase repository path uses today — does
// NOT overwrite the vector_store_id column, thanks to the GORM
// `<-:create` tag. This is the primary automated defense for the H1 review
// finding (silent overwrite of the binding via full-struct UPDATE).
func TestKnowledgeBase_VectorStoreID_Save_Immutable(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(kbStrPtr("store-A"))
require.NoError(t, db.Create(original).Error)
// Attempt to overwrite through the canonical Save() path.
mutated := *original
mutated.Name = "updated-name"
mutated.VectorStoreID = kbStrPtr("store-B")
require.NoError(t, db.Save(&mutated).Error)
reloaded := reloadKB(t, db, original.ID)
// Non-immutable fields must update.
assert.Equal(t, "updated-name", reloaded.Name, "Name must update through Save")
// Immutable field must NOT update.
require.NotNil(t, reloaded.VectorStoreID, "VectorStoreID must remain set to the original value")
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
"db.Save must not overwrite vector_store_id (enforced by GORM `<-:create` tag)")
}
// TestKnowledgeBase_VectorStoreID_Updates_Immutable verifies that
// db.Updates(struct{...}) also honors `<-:create` and silently ignores the
// vector_store_id column.
func TestKnowledgeBase_VectorStoreID_Updates_Immutable(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(kbStrPtr("store-A"))
require.NoError(t, db.Create(original).Error)
// Partial struct update that includes vector_store_id must not take effect.
err := db.Model(&types.KnowledgeBase{}).
Where("id = ?", original.ID).
Updates(types.KnowledgeBase{
Name: "updated-name",
VectorStoreID: kbStrPtr("store-B"),
}).Error
require.NoError(t, err)
reloaded := reloadKB(t, db, original.ID)
assert.Equal(t, "updated-name", reloaded.Name)
require.NotNil(t, reloaded.VectorStoreID)
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
"db.Updates must not overwrite vector_store_id (enforced by GORM `<-:create` tag)")
}
// TestKnowledgeBase_VectorStoreID_Select_StillImmutable verifies that even
// an explicit Select("vector_store_id").Updates(...) does NOT bypass the
// `<-:create` tag. GORM's `<-:create` is strict: it withholds write
// permission from every UPDATE path, including selective updates. This is
// a stronger guarantee than a typical "column is protected unless you
// opt in" scheme.
//
// The implication for Phase 4 (cross-store migration) is that ORM-level
// updates cannot move bindings. Migration code must use raw SQL (db.Exec)
// — see TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch below.
func TestKnowledgeBase_VectorStoreID_Select_StillImmutable(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(kbStrPtr("store-A"))
require.NoError(t, db.Create(original).Error)
err := db.Model(&types.KnowledgeBase{}).
Where("id = ?", original.ID).
Select("vector_store_id").
Updates(map[string]interface{}{"vector_store_id": "store-B"}).Error
require.NoError(t, err)
reloaded := reloadKB(t, db, original.ID)
require.NotNil(t, reloaded.VectorStoreID)
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
"Select(\"vector_store_id\").Updates must NOT bypass <-:create; ORM is fully locked")
}
// TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch documents that raw
// SQL UPDATE bypasses the GORM tag system entirely, as expected. Phase 4
// cross-store migration is the only sanctioned caller of this path, and
// it must do so through a dedicated, narrowly-scoped repository method
// rather than through general Save/Updates/Select flows.
func TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(kbStrPtr("store-A"))
require.NoError(t, db.Create(original).Error)
require.NoError(t,
db.Exec("UPDATE knowledge_bases SET vector_store_id = ? WHERE id = ?",
"store-B", original.ID).Error)
reloaded := reloadKB(t, db, original.ID)
require.NotNil(t, reloaded.VectorStoreID)
assert.Equal(t, "store-B", *reloaded.VectorStoreID,
"raw SQL UPDATE bypasses GORM tags; Phase 4 migration must use this path through a dedicated method")
}
// TestKnowledgeBase_VectorStoreID_Roundtrip_Nil verifies that a nil
// VectorStoreID is persisted as SQL NULL and loaded back as nil.
func TestKnowledgeBase_VectorStoreID_Roundtrip_Nil(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(nil)
require.NoError(t, db.Create(original).Error)
reloaded := reloadKB(t, db, original.ID)
assert.Nil(t, reloaded.VectorStoreID,
"nil VectorStoreID must round-trip through the database as nil")
// Extra assurance: column is actually NULL, not an empty string.
var nullCount int64
require.NoError(t,
db.Table("knowledge_bases").
Where("id = ? AND vector_store_id IS NULL", original.ID).
Count(&nullCount).Error)
assert.Equal(t, int64(1), nullCount, "nil must be stored as SQL NULL")
}
// TestKnowledgeBase_VectorStoreID_Roundtrip_Value verifies that a non-nil
// VectorStoreID is persisted and loaded back with the same value.
func TestKnowledgeBase_VectorStoreID_Roundtrip_Value(t *testing.T) {
db := setupKBTestDB(t)
original := makeKB(kbStrPtr("store-uuid-42"))
require.NoError(t, db.Create(original).Error)
reloaded := reloadKB(t, db, original.ID)
require.NotNil(t, reloaded.VectorStoreID)
assert.Equal(t, "store-uuid-42", *reloaded.VectorStoreID)
}
+7
View File
@@ -66,6 +66,13 @@ type KnowledgeBase struct {
StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"`
// Deprecated: legacy COS config column. Kept for backward compatibility with old data.
StorageConfig StorageConfig `yaml:"-" json:"storage_config" gorm:"column:cos_config;type:json"`
// VectorStoreID references the VectorStore this knowledge base is bound to.
// When nil, the KB falls back to the tenant's effective engines derived from
// the RETRIEVE_DRIVER environment variable (env store flow).
// This field is set once at creation time and must not be modified afterwards;
// enforcement lives at the GORM layer (`<-:create`) plus the service-layer
// KB update path, which omits this field from its update DTO.
VectorStoreID *string `yaml:"vector_store_id" json:"vector_store_id,omitempty" gorm:"column:vector_store_id;type:varchar(36);<-:create"`
// Extract config
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
// FAQConfig stores FAQ specific configuration such as indexing strategy
+104 -1
View File
@@ -1,6 +1,10 @@
package types
import "testing"
import (
"encoding/json"
"strings"
"testing"
)
func TestParseProviderScheme(t *testing.T) {
tests := []struct {
@@ -53,3 +57,102 @@ func TestInferStorageFromFilePath(t *testing.T) {
})
}
}
// strPtr returns a pointer to the given string, used to express *string literals in tests.
func strPtr(s string) *string { return &s }
// TestKnowledgeBase_VectorStoreID_JSON covers (un)marshaling of the nullable
// VectorStoreID pointer, including the `omitempty` behavior and the three
// distinct JSON inputs (missing field, explicit null, explicit value).
//
// The `empty string pointer` case is fixed here as a behavior snapshot: PR1
// accepts it and stores &"" verbatim because validation has not been added
// yet. A follow-up PR that introduces validation will convert this case into
// a rejection and flip the expectation in this test accordingly.
func TestKnowledgeBase_VectorStoreID_JSON(t *testing.T) {
t.Run("marshal nil omits field (omitempty)", func(t *testing.T) {
kb := KnowledgeBase{ID: "kb-1", VectorStoreID: nil}
b, err := json.Marshal(kb)
if err != nil {
t.Fatalf("json.Marshal returned error: %v", err)
}
if strings.Contains(string(b), `"vector_store_id"`) {
t.Errorf("expected vector_store_id to be omitted when nil, got: %s", b)
}
})
t.Run("marshal non-nil includes value", func(t *testing.T) {
kb := KnowledgeBase{ID: "kb-1", VectorStoreID: strPtr("store-uuid")}
b, err := json.Marshal(kb)
if err != nil {
t.Fatalf("json.Marshal returned error: %v", err)
}
if !strings.Contains(string(b), `"vector_store_id":"store-uuid"`) {
t.Errorf("expected vector_store_id to be serialized, got: %s", b)
}
})
unmarshalCases := []struct {
name string
body string
wantNil bool
wantValue string
}{
{name: "missing field", body: `{"id":"kb-1"}`, wantNil: true},
{name: "explicit null", body: `{"id":"kb-1","vector_store_id":null}`, wantNil: true},
{name: "explicit value", body: `{"id":"kb-1","vector_store_id":"store-uuid"}`, wantValue: "store-uuid"},
{name: "empty string pointer", body: `{"id":"kb-1","vector_store_id":""}`, wantValue: ""},
}
for _, tt := range unmarshalCases {
t.Run(tt.name, func(t *testing.T) {
var kb KnowledgeBase
if err := json.Unmarshal([]byte(tt.body), &kb); err != nil {
t.Fatalf("json.Unmarshal returned error: %v", err)
}
if tt.wantNil {
if kb.VectorStoreID != nil {
t.Errorf("expected VectorStoreID to be nil, got &%q", *kb.VectorStoreID)
}
return
}
if kb.VectorStoreID == nil {
t.Fatalf("expected VectorStoreID to be &%q, got nil", tt.wantValue)
}
if *kb.VectorStoreID != tt.wantValue {
t.Errorf("expected VectorStoreID = %q, got %q", tt.wantValue, *kb.VectorStoreID)
}
})
}
}
// TestKnowledgeBase_UnmarshalJSON_WithVectorStoreID verifies that the custom
// UnmarshalJSON on KnowledgeBase (which shadows cos_config for legacy
// compatibility) still delegates the new vector_store_id field to the alias
// type path, without interfering with StorageProviderConfig inference.
func TestKnowledgeBase_UnmarshalJSON_WithVectorStoreID(t *testing.T) {
// Legacy cos_config + new vector_store_id in the same payload: both must map correctly.
body := `{
"id": "kb-1",
"cos_config": {"provider": "cos", "bucket_name": "legacy-bucket"},
"vector_store_id": "store-uuid"
}`
var kb KnowledgeBase
if err := json.Unmarshal([]byte(body), &kb); err != nil {
t.Fatalf("json.Unmarshal returned error: %v", err)
}
if kb.VectorStoreID == nil || *kb.VectorStoreID != "store-uuid" {
t.Errorf("expected VectorStoreID = &\"store-uuid\", got %v", kb.VectorStoreID)
}
if kb.StorageConfig.Provider != "cos" {
t.Errorf("expected legacy StorageConfig.Provider = cos, got %q", kb.StorageConfig.Provider)
}
if kb.StorageProviderConfig == nil || kb.StorageProviderConfig.Provider != "cos" {
t.Errorf("expected StorageProviderConfig.Provider auto-populated from cos_config, got %v", kb.StorageProviderConfig)
}
// Regression guard: the aux struct inside UnmarshalJSON must not shadow vector_store_id.
// If a future change introduces such a shadow, the value above would fail to populate.
}
+3
View File
@@ -67,12 +67,15 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
is_pinned INTEGER NOT NULL DEFAULT 0,
pinned_at DATETIME NULL,
asr_config TEXT,
vector_store_id VARCHAR(36),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store
ON knowledge_bases(tenant_id, vector_store_id);
CREATE TABLE IF NOT EXISTS knowledges (
id VARCHAR(36) PRIMARY KEY,
@@ -0,0 +1,30 @@
-- Migration: 000036_kb_vector_store_id (rollback)
-- Description: Remove vector_store_id column and its index from knowledge_bases.
--
-- WARNING: This rollback is destructive. It permanently drops the vector_store_id column
-- and all KB ↔ VectorStore bindings. To preserve bindings before rollback:
--
-- CREATE TABLE knowledge_bases_vsid_backup AS
-- SELECT id, tenant_id, vector_store_id FROM knowledge_bases
-- WHERE vector_store_id IS NOT NULL;
--
-- This down migration does NOT block execution when non-NULL rows exist, because blocking
-- during an emergency rollback can make an incident worse. The RAISE NOTICE below logs the
-- number of rows whose bindings will be lost so operators can reconstruct later if needed.
DO $$
DECLARE non_null_count BIGINT;
BEGIN
SELECT COUNT(*) INTO non_null_count
FROM knowledge_bases WHERE vector_store_id IS NOT NULL;
IF non_null_count > 0 THEN
RAISE NOTICE '[Migration 000036 DOWN] Dropping vector_store_id with % non-NULL rows; these bindings will be lost', non_null_count;
ELSE
RAISE NOTICE '[Migration 000036 DOWN] Removing vector_store_id column from knowledge_bases';
END IF;
END $$;
DROP INDEX IF EXISTS idx_knowledge_bases_tenant_vector_store;
ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS vector_store_id;
DO $$ BEGIN RAISE NOTICE '[Migration 000036 DOWN] vector_store_id column removed successfully'; END $$;
@@ -0,0 +1,39 @@
-- Migration: 000036_kb_vector_store_id
-- Description: Add vector_store_id column to knowledge_bases for per-KB VectorStore binding.
-- NULL means "use tenant's effective engines" (env store derived from RETRIEVE_DRIVER),
-- which preserves existing behavior for all pre-existing rows.
-- No foreign key: env store IDs are virtual (not present in vector_stores), and
-- VectorStore soft-delete must not cascade into KB references. Referential integrity is
-- enforced at the application layer.
-- On PostgreSQL 11+ this ALTER runs as metadata-only (O(1), no row rewrite) because the
-- column is nullable with no DEFAULT. PostgreSQL 10 and earlier are not supported.
DO $$ BEGIN RAISE NOTICE '[Migration 000036] Adding vector_store_id column to knowledge_bases'; END $$;
ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS vector_store_id VARCHAR(36);
COMMENT ON COLUMN knowledge_bases.vector_store_id IS
'References vector_stores.id. NULL means tenant default (env store derived from RETRIEVE_DRIVER). '
'Immutable after creation (enforced at ORM and service layer). No FK by design.';
-- Composite index: tenant_id first to align with the dominant query pattern. All KB queries
-- start with `WHERE tenant_id = $1`, so the composite index enforces tenant-scoped plans and
-- hardens against accidental cross-tenant access. A full (non-partial) index is used because
-- the target end-state has most KBs bound to a store, making a partial index short-lived.
-- Storage cost is negligible at typical KB row counts.
CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_vector_store
ON knowledge_bases(tenant_id, vector_store_id);
-- Note: CREATE INDEX here holds a SHARE lock briefly. For very large knowledge_bases tables
-- (hundreds of thousands of rows or more), operators may prefer CREATE INDEX CONCURRENTLY.
-- This requires running the statement outside a transaction, which the standard migration
-- runner does not support; execute the CONCURRENTLY variant from a separate operational
-- script in that case. Simply replacing the statement below with CONCURRENTLY inside this
-- file will cause the migration to fail and leave schema_migrations in a dirty state.
DO $$
DECLARE non_null_count BIGINT;
BEGIN
SELECT COUNT(*) INTO non_null_count
FROM knowledge_bases WHERE vector_store_id IS NOT NULL;
RAISE NOTICE '[Migration 000036] knowledge_bases.vector_store_id added (% non-NULL rows)', non_null_count;
END $$;