c1f46df0dd
- Replaced `vlm_model_id` with a structured `vlm_config` object in knowledge base configurations, including `enabled` flag and `model_id`. - Updated API documentation to reflect changes in VLM configuration structure. - Enhanced frontend interfaces to accommodate new VLM configuration format. - Improved localization for new features related to knowledge graph and multimodal capabilities.
259 lines
10 KiB
SQL
259 lines
10 KiB
SQL
-- ============================================================================
|
|
-- Migration: Cleanup Unreferenced Models (PostgreSQL/ParadeDB)
|
|
-- Description: Remove models that are not referenced by any knowledge base
|
|
-- Created: 2025-11-03
|
|
-- ============================================================================
|
|
|
|
-- This script removes models from the models table that meet ALL of these conditions:
|
|
-- 1. Not referenced by any knowledge_bases (embedding_model_id, summary_model_id, rerank_model_id, vlm_config.model_id)
|
|
-- 2. Not referenced by any knowledges (embedding_model_id)
|
|
-- 3. Not a default model (is_default = false)
|
|
-- 4. Not a system model (tenant_id != 0)
|
|
-- 5. Not soft-deleted (deleted_at IS NULL)
|
|
|
|
-- WARNING: This operation is irreversible. Make sure to backup your database before running.
|
|
-- Recommended: Run the SELECT query first to review which models will be deleted.
|
|
|
|
-- ============================================================================
|
|
-- Step 1: Review models that will be deleted (DRY RUN)
|
|
-- ============================================================================
|
|
SELECT
|
|
m.id,
|
|
m.tenant_id,
|
|
m.name,
|
|
m.type,
|
|
m.source,
|
|
m.is_default,
|
|
m.status,
|
|
m.created_at
|
|
FROM models m
|
|
WHERE m.deleted_at IS NULL
|
|
AND m.is_default = FALSE
|
|
AND m.id NOT IN (
|
|
-- Models referenced by knowledge_bases
|
|
SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT (vlm_config ->> 'model_id') AS vlm_model_id
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND vlm_config ->> 'model_id' IS NOT NULL
|
|
AND vlm_config ->> 'model_id' != ''
|
|
UNION
|
|
-- Models referenced by knowledges
|
|
SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
)
|
|
ORDER BY m.created_at DESC;
|
|
|
|
-- ============================================================================
|
|
-- Step 2: Soft delete unreferenced models (set deleted_at timestamp)
|
|
-- ============================================================================
|
|
-- Uncomment the following line to perform soft delete:
|
|
-- UPDATE models m
|
|
-- SET deleted_at = CURRENT_TIMESTAMP
|
|
-- WHERE m.deleted_at IS NULL
|
|
-- AND m.is_default = FALSE
|
|
-- AND m.tenant_id != 0
|
|
-- AND m.id NOT IN (
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT (vlm_config ->> 'model_id') AS vlm_model_id
|
|
-- FROM knowledge_bases
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND vlm_config ->> 'model_id' IS NOT NULL
|
|
-- AND vlm_config ->> 'model_id' != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
-- );
|
|
|
|
-- ============================================================================
|
|
-- Step 3: Hard delete unreferenced models (PERMANENT DELETION)
|
|
-- ============================================================================
|
|
-- WARNING: This is permanent and cannot be undone!
|
|
-- Only use this if you're sure you want to permanently remove the records.
|
|
-- Uncomment the following line to perform hard delete:
|
|
-- DELETE FROM models
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND is_default = FALSE
|
|
-- AND tenant_id != 0
|
|
-- AND id NOT IN (
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT (vlm_config ->> 'model_id') AS vlm_model_id
|
|
-- FROM knowledge_bases
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND vlm_config ->> 'model_id' IS NOT NULL
|
|
-- AND vlm_config ->> 'model_id' != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
-- );
|
|
|
|
-- ============================================================================
|
|
-- Alternative: Using CTE for better performance (PostgreSQL specific)
|
|
-- ============================================================================
|
|
-- This version uses Common Table Expressions for better readability and performance
|
|
|
|
-- DRY RUN with CTE:
|
|
WITH referenced_models AS (
|
|
SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION
|
|
SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION
|
|
SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION
|
|
SELECT vlm_config ->> 'model_id'
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND vlm_config ->> 'model_id' IS NOT NULL
|
|
AND vlm_config ->> 'model_id' != ''
|
|
UNION
|
|
SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
)
|
|
SELECT
|
|
m.id,
|
|
m.tenant_id,
|
|
m.name,
|
|
m.type,
|
|
m.source,
|
|
m.is_default,
|
|
m.status,
|
|
m.created_at
|
|
FROM models m
|
|
WHERE m.deleted_at IS NULL
|
|
AND m.is_default = FALSE
|
|
AND m.id NOT IN (SELECT model_id FROM referenced_models)
|
|
ORDER BY m.created_at DESC;
|
|
|
|
-- Soft delete with CTE:
|
|
WITH referenced_models AS (
|
|
SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION
|
|
SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION
|
|
SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION
|
|
SELECT vlm_config ->> 'model_id'
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND vlm_config ->> 'model_id' IS NOT NULL
|
|
AND vlm_config ->> 'model_id' != ''
|
|
UNION
|
|
SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
)
|
|
UPDATE models m
|
|
SET deleted_at = CURRENT_TIMESTAMP
|
|
WHERE m.deleted_at IS NULL
|
|
AND m.is_default = FALSE
|
|
AND m.id NOT IN (SELECT model_id FROM referenced_models);
|
|
|
|
-- Hard delete with CTE:
|
|
-- WITH referenced_models AS (
|
|
-- SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
-- UNION
|
|
-- SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
-- UNION
|
|
-- SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
-- UNION
|
|
-- SELECT vlm_config ->> 'model_id'
|
|
-- FROM knowledge_bases
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND vlm_config ->> 'model_id' IS NOT NULL
|
|
-- AND vlm_config ->> 'model_id' != ''
|
|
-- UNION
|
|
-- SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
-- )
|
|
-- DELETE FROM models
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND is_default = FALSE
|
|
-- AND tenant_id != 0
|
|
-- AND id NOT IN (SELECT model_id FROM referenced_models);
|
|
|
|
-- ============================================================================
|
|
-- Additional Queries for Statistics
|
|
-- ============================================================================
|
|
|
|
-- Count total models by type
|
|
SELECT
|
|
type,
|
|
COUNT(*) as total_count,
|
|
SUM(CASE WHEN is_default = TRUE THEN 1 ELSE 0 END) as default_count,
|
|
SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active_count
|
|
FROM models
|
|
GROUP BY type;
|
|
|
|
-- Count models referenced by knowledge bases
|
|
SELECT
|
|
'embedding_model' as model_type,
|
|
COUNT(DISTINCT embedding_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'summary_model' as model_type,
|
|
COUNT(DISTINCT summary_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'rerank_model' as model_type,
|
|
COUNT(DISTINCT rerank_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'vlm_model' as model_type,
|
|
COUNT(DISTINCT (vlm_config ->> 'model_id')) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND vlm_config ->> 'model_id' IS NOT NULL
|
|
AND vlm_config ->> 'model_id' != '';
|
|
|
|
-- List all referenced models with their reference count
|
|
WITH model_references AS (
|
|
SELECT embedding_model_id AS model_id, 'embedding' AS ref_type FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION ALL
|
|
SELECT summary_model_id, 'summary' FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION ALL
|
|
SELECT rerank_model_id, 'rerank' FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION ALL
|
|
SELECT vlm_config ->> 'model_id', 'vlm'
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND vlm_config ->> 'model_id' IS NOT NULL
|
|
AND vlm_config ->> 'model_id' != ''
|
|
UNION ALL
|
|
SELECT embedding_model_id, 'knowledge_embedding' FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
)
|
|
SELECT
|
|
m.id,
|
|
m.name,
|
|
m.type,
|
|
STRING_AGG(DISTINCT mr.ref_type, ', ' ORDER BY mr.ref_type) AS referenced_as,
|
|
COUNT(*) AS reference_count
|
|
FROM models m
|
|
INNER JOIN model_references mr ON m.id = mr.model_id
|
|
WHERE m.deleted_at IS NULL
|
|
GROUP BY m.id, m.name, m.type
|
|
ORDER BY reference_count DESC, m.name;
|
|
|
|
-- ============================================================================
|
|
-- Rollback Strategy
|
|
-- ============================================================================
|
|
-- If you performed a soft delete and need to rollback:
|
|
-- UPDATE models
|
|
-- SET deleted_at = NULL
|
|
-- WHERE deleted_at > 'YOUR_DELETION_TIMESTAMP';
|
|
-- ============================================================================
|
|
|