c1f46df0dd
- Replaced `vlm_model_id` with a structured `vlm_config` object in knowledge base configurations, including `enabled` flag and `model_id`. - Updated API documentation to reflect changes in VLM configuration structure. - Enhanced frontend interfaces to accommodate new VLM configuration format. - Improved localization for new features related to knowledge graph and multimodal capabilities.
152 lines
6.6 KiB
SQL
152 lines
6.6 KiB
SQL
-- ============================================================================
|
|
-- Migration: Cleanup Unreferenced Models
|
|
-- Description: Remove models that are not referenced by any knowledge base
|
|
-- Created: 2025-11-03
|
|
-- ============================================================================
|
|
|
|
-- This script removes models from the models table that meet ALL of these conditions:
|
|
-- 1. Not referenced by any knowledge_bases (embedding_model_id, summary_model_id, rerank_model_id, vlm_config.model_id)
|
|
-- 2. Not referenced by any knowledges (embedding_model_id)
|
|
-- 3. Not a default model (is_default = false)
|
|
-- 4. Not a system model (tenant_id != 0)
|
|
-- 5. Not soft-deleted (deleted_at IS NULL)
|
|
|
|
-- WARNING: This operation is irreversible. Make sure to backup your database before running.
|
|
-- Recommended: Run the SELECT query first to review which models will be deleted.
|
|
|
|
-- ============================================================================
|
|
-- Step 1: Review models that will be deleted (DRY RUN)
|
|
-- ============================================================================
|
|
SELECT
|
|
m.id,
|
|
m.tenant_id,
|
|
m.name,
|
|
m.type,
|
|
m.source,
|
|
m.is_default,
|
|
m.status,
|
|
m.created_at
|
|
FROM models m
|
|
WHERE m.deleted_at IS NULL
|
|
AND m.is_default = FALSE
|
|
AND m.tenant_id != 0
|
|
AND m.id NOT IN (
|
|
-- Models referenced by knowledge_bases
|
|
SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION
|
|
SELECT DISTINCT JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) AS vlm_model_id
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND JSON_EXTRACT(vlm_config, '$.model_id') IS NOT NULL
|
|
AND JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) != ''
|
|
UNION
|
|
-- Models referenced by knowledges
|
|
SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
)
|
|
ORDER BY m.created_at DESC;
|
|
|
|
-- ============================================================================
|
|
-- Step 2: Soft delete unreferenced models (set deleted_at timestamp)
|
|
-- ============================================================================
|
|
-- Uncomment the following line to perform soft delete:
|
|
-- UPDATE models m
|
|
-- SET deleted_at = CURRENT_TIMESTAMP
|
|
-- WHERE m.deleted_at IS NULL
|
|
-- AND m.is_default = FALSE
|
|
-- AND m.tenant_id != 0
|
|
-- AND m.id NOT IN (
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) AS vlm_model_id
|
|
-- FROM knowledge_bases
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND JSON_EXTRACT(vlm_config, '$.model_id') IS NOT NULL
|
|
-- AND JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
-- );
|
|
|
|
-- ============================================================================
|
|
-- Step 3: Hard delete unreferenced models (PERMANENT DELETION)
|
|
-- ============================================================================
|
|
-- WARNING: This is permanent and cannot be undone!
|
|
-- Only use this if you're sure you want to permanently remove the records.
|
|
-- Uncomment the following line to perform hard delete:
|
|
-- DELETE FROM models
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND is_default = FALSE
|
|
-- AND tenant_id != 0
|
|
-- AND id NOT IN (
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) AS vlm_model_id
|
|
-- FROM knowledge_bases
|
|
-- WHERE deleted_at IS NULL
|
|
-- AND JSON_EXTRACT(vlm_config, '$.model_id') IS NOT NULL
|
|
-- AND JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) != ''
|
|
-- UNION
|
|
-- SELECT DISTINCT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != ''
|
|
-- );
|
|
|
|
-- ============================================================================
|
|
-- Additional Queries for Statistics
|
|
-- ============================================================================
|
|
|
|
-- Count total models by type
|
|
SELECT
|
|
type,
|
|
COUNT(*) as total_count,
|
|
SUM(CASE WHEN is_default = TRUE THEN 1 ELSE 0 END) as default_count,
|
|
SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) as active_count
|
|
FROM models
|
|
GROUP BY type;
|
|
|
|
-- Count models referenced by knowledge bases
|
|
SELECT
|
|
'embedding_model' as model_type,
|
|
COUNT(DISTINCT embedding_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND embedding_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'summary_model' as model_type,
|
|
COUNT(DISTINCT summary_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND summary_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'rerank_model' as model_type,
|
|
COUNT(DISTINCT rerank_model_id) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL AND rerank_model_id != ''
|
|
UNION ALL
|
|
SELECT
|
|
'vlm_model' as model_type,
|
|
COUNT(DISTINCT JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id'))) as referenced_count
|
|
FROM knowledge_bases
|
|
WHERE deleted_at IS NULL
|
|
AND JSON_EXTRACT(vlm_config, '$.model_id') IS NOT NULL
|
|
AND JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) != '';
|
|
|
|
-- ============================================================================
|
|
-- Rollback Strategy
|
|
-- ============================================================================
|
|
-- If you performed a soft delete and need to rollback:
|
|
-- UPDATE models
|
|
-- SET deleted_at = NULL
|
|
-- WHERE deleted_at > 'YOUR_DELETION_TIMESTAMP';
|
|
-- ============================================================================
|
|
|