feat(sources): 数据源配置收敛成 dws 导入 + PP 导入;「去掉来源」= 归档下架(不删数据)
build / build (push) Successful in 58s

主人要求:去掉入口,统一成 dws 导入与 PP 导入;其他来源连数据一起下架,之后用 dws 重新获取。

界面(web):
- 「数据源配置」只剩两块:**dws 导入(钉钉)**(按当前钉钉账号列出可访问的知识库与钉盘空间)
  与 **PP 导入**(PP 资料下载中心)。上一步加的「显示 AList/钉钉知识库来源」折叠开关删除。
- 来源卡片新增管理员可见的「去掉来源」(带确认弹窗,说明是归档下架、数据保留可恢复)。
- 侧栏文案改成"只留两个入口";发现面板标题改为「dws 导入(钉钉)」。

后端(归档语义,可逆):
- 迁移 000047:source_connections 增 archived_at / archived_by + 只覆盖未归档行的部分索引。
- store:ArchiveSourceConnection / RestoreSourceConnection(幂等;归档同时停用并清空
  next_refresh_at,避免被手工改回后立刻带着旧调度跑)。列表、目录、计划刷新(claimDue)、
  手动扫描(triggerBlock/inspect)四类查询一律排除归档行。
- 注册路径(dws 重新获取):RegisterDingtalkSpace 的 ON CONFLICT 清掉归档标记 ——
  重新登记同一个空间**原地复活同一行**,不产生重复,历史快照仍属于同一行。
- HTTP:POST /api/v1/source-connections/{id}/archive|restore(管理员专属,写意图头
  source-archive / source-restore + Idempotency-Key;404 区分"不存在"与"已是目标状态");
  openapi 同步两条契约。

验证:
- 集成测试(真 postgres):归档从列表消失但 source_items 一行不少、重复归档不漂移归档时间与人、
  未知 ID 报 ErrSourceConnectionNotFound、归档来源不能再被扫描、恢复后回到列表但保持停用;
  dws 重新登记同一空间复活同一行。
- httpapi:写意图/幂等键门禁、200/403/404/500、错误体不回显内部细节、classifyRole 管理员专属。
- 迁移护栏:000047 结构 + 四类 SQL 必须含 archived_at is null(漏一条就是入口没去掉)。
- make verify 15/15。
This commit is contained in:
姚章浩
2026-10-14 12:36:32 +08:00
parent 64b32c2da6
commit 2c7265efc4
20 changed files with 1426 additions and 281 deletions
@@ -80,6 +80,9 @@ INSERT INTO source_connections (
) VALUES ($1, 'dingtalk', $2, $3, '', TRUE, '{}'::jsonb, '{}'::jsonb, $4, $4, $5)
ON CONFLICT (id) DO UPDATE
SET name = EXCLUDED.name, base_url = EXCLUDED.base_url,
-- 归档过的来源被重新登记(dws 重新获取)时原地复活:清掉归档标记,不建重复行 ——
-- 目录快照与历史仍属于同一行。
archived_at = NULL, archived_by = '',
updated_at = EXCLUDED.updated_at, updated_by = EXCLUDED.updated_by`,
connectionID, in.Name, locator, in.Now.UTC(), "dingtalk-registration")
if err != nil {
@@ -0,0 +1,7 @@
DROP INDEX IF EXISTS idx_source_connections_active;
ALTER TABLE source_connections
DROP COLUMN IF EXISTS archived_by;
ALTER TABLE source_connections
DROP COLUMN IF EXISTS archived_at;
@@ -0,0 +1,20 @@
-- 「去掉来源」= 归档下架,而不是物理删除。
--
-- 归档后这个来源会从所有列表与调度里消失:数据源配置不再列它、来源与目录不再出现
-- 它的快照、计划刷新不会再拉它、手动扫描会被拒。目录快照、发布记录与审计数据
-- **原样保留**;恢复只需把 archived_at 置空,或让 dws 重新获取同一个空间
-- (钉钉空间登记用确定性 id + ON CONFLICT,会原地复活同一行,不会产生重复)。
--
-- 为什么不物理删除:source_items / source_catalog_snapshots / test_publications /
-- release_* 都以 ON DELETE RESTRICT 引用它,硬删要么失败,要么连带毁掉发布与审计
-- 记录 —— 而这两样正是治理平台存在的理由。
ALTER TABLE source_connections
ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
ALTER TABLE source_connections
ADD COLUMN IF NOT EXISTS archived_by VARCHAR(255) NOT NULL DEFAULT '';
-- 列表、调度、目录都只关心未归档的行:部分索引让这些查询继续走索引。
CREATE INDEX IF NOT EXISTS idx_source_connections_active
ON source_connections (source_type, name)
WHERE archived_at IS NULL;
+23 -2
View File
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 46 {
t.Fatalf("migration count = %d, want 46", len(migrations))
if len(migrations) != 47 {
t.Fatalf("migration count = %d, want 47", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -125,6 +125,27 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if !strings.Contains(migrations[45].DownSQL, "DROP TABLE IF EXISTS version_history_commits") {
t.Fatalf("migration 46 down = %#v", migrations[45].DownSQL)
}
// 000047:「去掉来源」= 归档下架而不是物理删除(内容、发布与审计都靠 ON DELETE
// RESTRICT 引用来源连接,硬删要么失败要么毁证据)。归档标记 + 只覆盖未归档行的部分索引。
if migrations[46].Version != 47 || !strings.Contains(migrations[46].UpSQL, "ADD COLUMN IF NOT EXISTS archived_at") ||
!strings.Contains(migrations[46].UpSQL, "ADD COLUMN IF NOT EXISTS archived_by") ||
!strings.Contains(migrations[46].UpSQL, "WHERE archived_at IS NULL") {
t.Fatalf("migration 47 = %#v", migrations[46])
}
if !strings.Contains(migrations[46].DownSQL, "DROP COLUMN IF EXISTS archived_at") {
t.Fatalf("migration 47 down = %#v", migrations[46].DownSQL)
}
// 归档行必须在每一条「来源可见 / 可调度」的 SQL 里被排除;漏一条就等于入口没去掉。
for name, query := range map[string]string{
"listSourceConnectionsSQL": listSourceConnectionsSQL,
"claimDueSourceRefreshesSQL": claimDueSourceRefreshesSQL,
"triggerPPSourceScanSQL": triggerPPSourceScanSQL,
"inspectPPSourceScanBlockSQL": inspectPPSourceScanBlockSQL,
} {
if !strings.Contains(strings.ToLower(query), "archived_at is null") {
t.Errorf("%s must exclude archived connections", name)
}
}
}
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
+2 -1
View File
@@ -35,7 +35,7 @@ JOIN LATERAL (
LIMIT 1
) job ON TRUE
LEFT JOIN source_catalog_heads head ON head.source_connection_id=connection.id
WHERE connection.enabled AND head.source_connection_id IS NULL
WHERE connection.enabled AND connection.archived_at IS NULL AND head.source_connection_id IS NULL
ORDER BY job.created_at DESC, connection.id`)
if err != nil {
return nil, fmt.Errorf("list catalog scan progress: %w", err)
@@ -90,6 +90,7 @@ LEFT JOIN LATERAL (
ORDER BY job.completed_at DESC
LIMIT 1
) refresh ON TRUE
WHERE connection.archived_at IS NULL
ORDER BY CASE connection.source_type WHEN 'dingtalk' THEN 1 WHEN 'pp' THEN 2 WHEN 'alist' THEN 3 ELSE 4 END,
snapshot.source_label, connection.id`)
if err != nil {
@@ -0,0 +1,223 @@
package postgres
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// 「去掉来源」的语义护栏:归档之后来源从列表里消失、不能再被手动扫描,
// 但**内容与审计数据一行都不少**;恢复/重新登记都能原地复活,且不产生重复行。
// 这些性质就是"可以不物理删除"的全部理由,所以逐条钉死。
func TestSourceConnectionArchiveHidesFromListsAndKeepsData(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("ApplyMigrations() error = %v", err)
}
store := NewStore(pool)
now := time.Date(2026, 10, 14, 4, 0, 0, 0, time.UTC)
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections (id, source_type, name, enabled, created_at, updated_at)
VALUES ('conn-keep-pp', 'pp', 'PP资料下载中心', TRUE, $1, $1),
('conn-drop-alist', 'alist', '国内工业', TRUE, $1, $1),
('conn-drop-pp', 'pp', 'PP legacy', TRUE, $1, $1)`, now); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO source_items (id, source_connection_id, stable_external_id, canonical_uri_hash,
title, item_type, visibility, visibility_origin, source_change, created_at, updated_at)
VALUES ('item-drop-1', 'conn-drop-alist', 'ext-1', repeat('a', 64), '要被归档的文档',
'file', 'internal', 'configured_scope', 'unchanged', $1, $1)`, now); err != nil {
t.Fatal(err)
}
page, err := store.ListSourceConnections(ctx)
if err != nil {
t.Fatalf("ListSourceConnections() error = %v", err)
}
if len(page.Items) != 3 {
t.Fatalf("前置状态:want 3 sources, got %d", len(page.Items))
}
if err := store.ArchiveSourceConnection(ctx, "conn-drop-alist", "yaozhanghao", now); err != nil {
t.Fatalf("ArchiveSourceConnection() error = %v", err)
}
page, err = store.ListSourceConnections(ctx)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 2 {
t.Fatalf("归档后仍列出:%#v", page.Items)
}
for _, item := range page.Items {
if item.ID == "conn-drop-alist" {
t.Fatalf("归档的来源不该出现在列表里:%#v", item)
}
}
// 内容数据必须原样保留:这是"归档而不是删除"的核心承诺。
var keptItems int
var archivedAt *time.Time
var archivedBy string
if err := pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM source_items WHERE source_connection_id='conn-drop-alist'`).Scan(&keptItems); err != nil {
t.Fatal(err)
}
if keptItems != 1 {
t.Fatalf("归档不该动内容数据:source_items=%d, want 1", keptItems)
}
if err := pool.QueryRow(ctx, `SELECT archived_at, archived_by FROM source_connections WHERE id='conn-drop-alist'`).Scan(&archivedAt, &archivedBy); err != nil {
t.Fatal(err)
}
if archivedAt == nil || !archivedAt.Equal(now) || archivedBy != "yaozhanghao" {
t.Fatalf("归档证据 = %v / %q", archivedAt, archivedBy)
}
// 幂等:重复归档不改归档时间与归档人(谁先归档就以谁为准)。
if err := store.ArchiveSourceConnection(ctx, "conn-drop-alist", "someone-else", now.Add(time.Hour)); err != nil {
t.Fatalf("重复归档该幂等成功:%v", err)
}
var again *time.Time
var againBy string
if err := pool.QueryRow(ctx, `SELECT archived_at, archived_by FROM source_connections WHERE id='conn-drop-alist'`).Scan(&again, &againBy); err != nil {
t.Fatal(err)
}
if again == nil || !again.Equal(now) || againBy != "yaozhanghao" {
t.Fatalf("重复归档漂移了归档证据 = %v / %q", again, againBy)
}
// 未知连接要能和"已经是目标状态"区分开。
if err := store.ArchiveSourceConnection(ctx, "conn-missing", "yaozhanghao", now); !errors.Is(err, ErrSourceConnectionNotFound) {
t.Fatalf("未知连接归档 = %v, want ErrSourceConnectionNotFound", err)
}
if err := store.RestoreSourceConnection(ctx, "conn-missing", "yaozhanghao", now); !errors.Is(err, ErrSourceConnectionNotFound) {
t.Fatalf("未知连接恢复 = %v, want ErrSourceConnectionNotFound", err)
}
// 归档来源不能再被手动扫描(否则"去掉入口"只是界面假象)。
if _, err := store.TriggerPPSourceScan(ctx, "conn-drop-alist", now); !errors.Is(err, ErrSourceConnectionNotScannable) {
t.Fatalf("归档来源扫描 = %v, want ErrSourceConnectionNotScannable", err)
}
if err := store.ArchiveSourceConnection(ctx, "conn-drop-pp", "yaozhanghao", now); err != nil {
t.Fatal(err)
}
if _, err := store.TriggerPPSourceScan(ctx, "conn-drop-pp", now); !errors.Is(err, ErrSourceConnectionNotScannable) {
t.Fatalf("归档的 PP 来源扫描 = %v, want ErrSourceConnectionNotScannable", err)
}
// 恢复:回到列表,但保持停用、调度为空 —— 归档时清掉的调度不悄悄回来。
if err := store.RestoreSourceConnection(ctx, "conn-drop-alist", "yaozhanghao", now); err != nil {
t.Fatalf("RestoreSourceConnection() error = %v", err)
}
page, err = store.ListSourceConnections(ctx)
if err != nil {
t.Fatal(err)
}
// conn-drop-pp 仍然归档中,只有 conn-drop-alist 回来了:2 = 3 - 仍归档的 1。
if len(page.Items) != 2 {
t.Fatalf("恢复后 want 2 sources, got %d", len(page.Items))
}
var restoredSeen bool
for _, item := range page.Items {
if item.ID == "conn-drop-pp" {
t.Fatalf("仍在归档中的来源不该出现:%#v", item)
}
if item.ID == "conn-drop-alist" {
restoredSeen = true
}
}
if !restoredSeen {
t.Fatalf("恢复的来源没有回到列表:%#v", page.Items)
}
var enabled bool
var nextRefresh *time.Time
var restoredArchived *time.Time
if err := pool.QueryRow(ctx, `SELECT enabled, next_refresh_at, archived_at FROM source_connections WHERE id='conn-drop-alist'`).
Scan(&enabled, &nextRefresh, &restoredArchived); err != nil {
t.Fatal(err)
}
if enabled || nextRefresh != nil || restoredArchived != nil {
t.Fatalf("恢复状态 = enabled:%v next:%v archived:%v", enabled, nextRefresh, restoredArchived)
}
// 未归档时恢复幂等成功(不是错误)。
if err := store.RestoreSourceConnection(ctx, "conn-drop-alist", "yaozhanghao", now); err != nil {
t.Fatalf("重复恢复该幂等成功:%v", err)
}
}
// 「等下用 dws 重新获取」必须原地复活同一个来源,而不是留下一堆重复行:
// 钉钉空间登记用确定性 id + ON CONFLICT,归档标记要在登记时被清掉。
func TestRegisterDingtalkSpaceRevivesArchivedConnection(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("ApplyMigrations() error = %v", err)
}
store := NewStore(pool)
now := time.Date(2026, 10, 14, 4, 0, 0, 0, time.UTC)
input := RegisterDingtalkSpaceInput{
RootKey: "revive-key", Name: "ATOM知识库", SpaceID: "space-revive", RootID: "root-revive", Now: now,
}
first, err := store.RegisterDingtalkSpace(ctx, input)
if err != nil {
t.Fatalf("RegisterDingtalkSpace() error = %v", err)
}
if err := store.ArchiveSourceConnection(ctx, first, "yaozhanghao", now); err != nil {
t.Fatal(err)
}
page, err := store.ListSourceConnections(ctx)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 0 {
t.Fatalf("归档后不该列出:%#v", page.Items)
}
second, err := store.RegisterDingtalkSpace(ctx, input)
if err != nil {
t.Fatalf("重新登记失败:%v", err)
}
if second != first {
t.Fatalf("重新登记产生了新行:%s → %s", first, second)
}
page, err = store.ListSourceConnections(ctx)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 || page.Items[0].ID != first {
t.Fatalf("重新获取后应当原地复活且不重复:%#v", page.Items)
}
var archivedAt *time.Time
var archivedBy string
var rows int
if err := pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM source_connections WHERE id = $1`, first).Scan(&rows); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT archived_at, archived_by FROM source_connections WHERE id = $1`, first).Scan(&archivedAt, &archivedBy); err != nil {
t.Fatal(err)
}
if rows != 1 || archivedAt != nil || archivedBy != "" {
t.Fatalf("复活状态 = rows:%d archived:%v by:%q", rows, archivedAt, archivedBy)
}
}
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"knowledge-governance/internal/governance"
@@ -47,6 +48,7 @@ LEFT JOIN LATERAL (
ORDER BY created_at DESC, id DESC
LIMIT 1
) scan ON TRUE
WHERE c.archived_at IS NULL
GROUP BY c.id, c.source_type, c.name, c.enabled, c.schedule, c.last_health, c.updated_at, c.next_refresh_at,
scan.id, scan.state, scan.observed_items, scan.unchanged_items,
scan.added_items, scan.updated_items, scan.over_limit_items,
@@ -158,7 +160,7 @@ WITH due AS (
FROM source_connections AS c
LEFT JOIN source_connection_revisions AS r
ON r.id = c.current_revision_id AND r.state = 'active'
WHERE c.enabled AND c.source_type <> 'manual'
WHERE c.enabled AND c.source_type <> 'manual' AND c.archived_at IS NULL
AND c.next_refresh_at IS NOT NULL AND c.next_refresh_at <= $1
ORDER BY c.next_refresh_at, c.id
FOR UPDATE OF c SKIP LOCKED
@@ -225,3 +227,64 @@ func NextRefreshAtFor(enabled bool, scheduleRaw []byte, from time.Time) (*time.T
next := from.UTC().Add(interval)
return &next, nil
}
// ArchiveSourceConnection 把来源连接从平台上「去掉入口」:置 archived_at 之后它会
// 从数据源配置、来源与目录、计划刷新与扫描里全部消失,但行、目录快照、发布记录与
// 审计原样保留 —— 恢复(RestoreSourceConnection 或重新登记同一来源)即可原地复活。
//
// 幂等:已归档时不改写归档时间与归档人,只返回成功(重复点击不漂移语义)。同时停用并
// 清空 next_refresh_at:即使有人手工把 archived_at 改回来,也不会带着旧调度立刻跑起来。
func (s *Store) ArchiveSourceConnection(ctx context.Context, connectionID, actor string, now time.Time) error {
if strings.TrimSpace(connectionID) == "" || now.IsZero() {
return errors.New("archive source connection: id and time are required")
}
if strings.TrimSpace(actor) == "" {
return errors.New("archive source connection: actor is required")
}
tag, err := s.db.Exec(ctx, `
UPDATE source_connections
SET archived_at = COALESCE(archived_at, $2),
archived_by = CASE WHEN archived_at IS NULL THEN $3 ELSE archived_by END,
enabled = FALSE,
next_refresh_at = NULL,
updated_at = $2,
updated_by = $3
WHERE id = $1`, connectionID, now.UTC(), actor)
if err != nil {
return fmt.Errorf("archive source connection: %w", err)
}
if tag.RowsAffected() == 0 {
return ErrSourceConnectionNotFound
}
return nil
}
// RestoreSourceConnection 撤销归档:来源重新出现在列表里,但**保持停用**,需要管理员
// 再显式启用(归档时被清掉的调度不会悄悄回来)。幂等:未归档时视为成功;只有连接真的
// 不存在才报 ErrSourceConnectionNotFound。
func (s *Store) RestoreSourceConnection(ctx context.Context, connectionID, actor string, now time.Time) error {
if strings.TrimSpace(connectionID) == "" || now.IsZero() {
return errors.New("restore source connection: id and time are required")
}
if strings.TrimSpace(actor) == "" {
return errors.New("restore source connection: actor is required")
}
tag, err := s.db.Exec(ctx, `
UPDATE source_connections
SET archived_at = NULL, archived_by = '', updated_at = $2, updated_by = $3
WHERE id = $1 AND archived_at IS NOT NULL`, connectionID, now.UTC(), actor)
if err != nil {
return fmt.Errorf("restore source connection: %w", err)
}
if tag.RowsAffected() > 0 {
return nil
}
var exists bool
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id = $1)`, connectionID).Scan(&exists); err != nil {
return fmt.Errorf("restore source connection: %w", err)
}
if !exists {
return ErrSourceConnectionNotFound
}
return nil
}
+2 -2
View File
@@ -28,7 +28,7 @@ const triggerPPSourceScanSQL = `
WITH target AS (
SELECT id, source_type
FROM source_connections
WHERE id = $1 AND source_type = 'pp'
WHERE id = $1 AND source_type = 'pp' AND archived_at IS NULL
), active AS (
SELECT 1
FROM source_scans
@@ -65,7 +65,7 @@ SELECT connection.source_type,
AND scan.state IN ('pending', 'running')
)
FROM source_connections AS connection
WHERE connection.id = $1
WHERE connection.id = $1 AND connection.archived_at IS NULL
`
const startSourceScanSQL = `