Files
knowledge-governance/internal/releaseadapter/v071/wiki_export.go
T
姚章浩 ea6e56b304 feat(release): 文档发布会带上知识库 wiki 快照并支持撤权剥离
发布通道原来只有三条:PostgreSQL 关系行、Elasticsearch 分片、OSS 对象。
WeKnora 的 wiki 图谱是 wiki_pages 一张纯 PG 表(in_links/out_links 存 slug),
所以它不在任何通道里——撤权后生产 wiki 会一直留着已撤权文档的页面与引用。

本次新增第 5 条通道:库级 wiki 快照,挂在现有文档发布包里,检索与 wiki
同一个 release 单元(同一次执行、同一次校验、同一次回滚):

- 导出:ExportWiki 按 knowledge_base_id 全量读 wiki_pages/wiki_folders;
  收敛门禁(库内无解析中/合成中文档)不满足直接失败并给中文原因;
  未启用 wiki 且无 wiki 行的库不参与同步(生产 wiki 保持原样)。
- 撤权:item.Action == "delete" 时在导出端剥离该文档的 wiki 贡献
  (source_refs/chunk_refs 去条目、无引用页面整页不带、摘死链、清 parent_slug、
  无页面引用的空文件夹及其祖先一并去掉),使“源端仍在只停止发布”和
  “源端删除”两种情况收敛到同一结果。
- 导入:库级 replace-by-kb(同一事务内 DELETE by kb 后灌入快照,文件夹先于页面),
  新增 ReplaceWiki 供撤权路径使用(文档行只 disable,wiki 整体替换)。
- 校验:VerifyWiki 用生产 WeKnora wiki API 比对页面 slug 集合,撤权时额外断言
  生产 wiki 不再引用被撤权文档。

驱动与验收:新增 cmd/release-channel-e2e(复用发布 worker 真实代码路径,
支持 publish/withdraw/rollback),scripts/wiki_e2e_test5.sh、
scripts/wiki_e2e_newkb.sh 两套端到端脚本;证据见
docs/WIKI_CO_TRANSPORT_ACCEPTANCE.md。
2026-09-30 10:54:07 +08:00

313 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"encoding/json"
"errors"
"strings"
)
// 库级 wiki 快照(第 5 条发布通道)。
//
// WeKnora 的 wiki 图谱不是 Neo4j 图谱:节点与边都在 wiki_pages 一张表里
// (in_links/out_links 存 slug 数组,库内相对引用),文件夹在 wiki_folders。
// 两者都是纯 PostgreSQL 数据,同一个 knowledge_base_id 可以直接搬运,
// 不需要重建,也不依赖 Neo4j / Elasticsearch。
//
// 快照粒度是“整库”,不是“单文档”:撤一篇文档会改动其它共贡页面的正文
// (retract 重写、摘死链、重建索引页),所以 delta 的最小单位只能是整库。
const exportWikiSettlementSQL = `
SELECT
(SELECT COUNT(*)
FROM knowledges AS knowledge
WHERE knowledge.tenant_id = $1 AND knowledge.knowledge_base_id = $2 AND knowledge.deleted_at IS NULL
AND (knowledge.pending_subtasks_count > 0 OR knowledge.parse_status IN ('pending', 'processing', 'finalizing'))),
(SELECT COUNT(*) FROM wiki_pages AS page WHERE page.tenant_id = $1 AND page.knowledge_base_id = $2),
(SELECT COUNT(*) FROM wiki_folders AS folder WHERE folder.tenant_id = $1 AND folder.knowledge_base_id = $2),
(SELECT COALESCE((base.indexing_strategy ->> 'wiki_enabled')::boolean, false)
FROM knowledge_bases AS base
WHERE base.id = $2 AND base.tenant_id = $1 AND base.deleted_at IS NULL)`
const exportWikiPagesSQL = `
SELECT to_jsonb(page)
FROM wiki_pages AS page
WHERE page.tenant_id = $1 AND page.knowledge_base_id = $2 AND page.deleted_at IS NULL
ORDER BY page.slug`
const exportWikiFoldersSQL = `
SELECT to_jsonb(folder)
FROM wiki_folders AS folder
WHERE folder.tenant_id = $1 AND folder.knowledge_base_id = $2 AND folder.deleted_at IS NULL
ORDER BY folder.id`
// wikiSnapshotExporter 由 PostgresExporter 实现;文档导出器在存在该能力时把
// 库级 wiki 快照追加进同一个 bundle,使检索与 wiki 落在同一个发布单元里。
type wikiSnapshotExporter interface {
ExportWiki(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string) ([]TableSnapshot, error)
}
// ExportWiki 导出该知识库的 wiki 快照。
//
// removingKnowledgeID 非空表示本次是撤权(删除)发布:剥掉该文档在 wiki 里的
// 贡献。源端“仅停用”不会改动 wiki(WeKnora 只在删除时跑 retract 清理链),
// 这一层剥离让“停止发布”和“源端删除”两种情况收敛到同一个结果:生产 wiki
// 不再引用该文档。源端真删除时清理链已经跑过,剥离是幂等空操作。
func (exporter *PostgresExporter) ExportWiki(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string) ([]TableSnapshot, error) {
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
return nil, errors.New("invalid wiki export identity")
}
if removingKnowledgeID != "" && !validSearchIdentifier(removingKnowledgeID) {
return nil, errors.New("invalid wiki removal identity")
}
var pending, pageCount, folderCount int
var wikiEnabled *bool
if err := exporter.db.QueryRow(ctx, exportWikiSettlementSQL, identity.TenantID, identity.KnowledgeBaseID).
Scan(&pending, &pageCount, &folderCount, &wikiEnabled); err != nil {
return nil, errors.New("read wiki snapshot state")
}
// 未启用 wiki 且库里没有任何 wiki 行的知识库不参与同步:包内不带 wiki 表,
// 生产端 wiki 保持原样,避免把生产 wiki 误清空。
if (wikiEnabled == nil || !*wikiEnabled) && pageCount == 0 && folderCount == 0 {
return nil, nil
}
// 收敛门禁:还有文档在解析或 wiki 合成中时不导出,避免产出半成品 wiki 快照。
if pending > 0 {
return nil, errors.New("源端 wiki 尚未收敛:该知识库仍有文档在解析或 wiki 合成中,请稍后重试")
}
pages, err := exporter.queryRows(ctx, exportWikiPagesSQL, identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return nil, err
}
folders, err := exporter.queryRows(ctx, exportWikiFoldersSQL, identity.TenantID, identity.KnowledgeBaseID)
if err != nil {
return nil, err
}
for _, page := range pages {
if !matchesWikiPageRow(page, identity) {
return nil, errors.New("PostgreSQL returned a cross-scope wiki page")
}
}
for _, folder := range folders {
if !matchesWikiFolderRow(folder, identity) {
return nil, errors.New("PostgreSQL returned a cross-scope wiki folder")
}
}
if removingKnowledgeID != "" {
pages, folders, err = exporter.retractWikiContribution(ctx, identity, removingKnowledgeID, pages, folders)
if err != nil {
return nil, err
}
}
// wiki_folders 先于 wiki_pages:页面通过 folder_id 引用文件夹。
return []TableSnapshot{
newTableSnapshot("wiki_folders", folders),
newTableSnapshot("wiki_pages", pages),
}, nil
}
// retractWikiContribution 从库级快照里剥掉一篇文档的 wiki 贡献:
// - source_refs 形如 "<knowledge_id>|<doc_title>",剥掉该 knowledge 的条目;
// - chunk_refs 是该文档 chunk UUID,剥掉属于该 knowledge 的条目;
// - 剥完没有任何来源引用的页面整页不带走(等同源端“专属页删除”);
// 索引页 / 日志页本来就没有 source_refs,保留;
// - 指向被丢弃页面的 in_links / out_links 一并摘掉(避免生产死链);
// - 父页被丢弃时清空 parent_slug;
// - 没有页面再引用的文件夹(以及它们的空祖先)也一并去掉,避免生产 wiki
// 留下“空目录”这种撤权痕迹。
func (exporter *PostgresExporter) retractWikiContribution(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string, pages, folders []json.RawMessage) ([]json.RawMessage, []json.RawMessage, error) {
chunks, err := exporter.queryRows(ctx, exportChunksSQL, removingKnowledgeID, identity.TenantID)
if err != nil {
return nil, nil, err
}
removedChunks := make(map[string]bool, len(chunks))
for _, chunk := range chunks {
chunkID, ok := matchesChunkRow(chunk, identity)
if !ok {
return nil, nil, errors.New("PostgreSQL returned a cross-scope chunk row")
}
removedChunks[chunkID] = true
}
kept := make([]json.RawMessage, 0, len(pages))
droppedSlugs := map[string]bool{}
pending := make([]map[string]any, 0, len(pages))
keptFolderIDs := map[string]bool{}
for _, raw := range pages {
page, err := decodeWikiRow(raw)
if err != nil {
return nil, nil, err
}
sourceRefs := stringList(page["source_refs"])
chunkRefs := stringList(page["chunk_refs"])
referenceBearing := len(sourceRefs) > 0 || len(chunkRefs) > 0
keptSourceRefs := make([]string, 0, len(sourceRefs))
for _, reference := range sourceRefs {
if !referencesKnowledge(reference, removingKnowledgeID) {
keptSourceRefs = append(keptSourceRefs, reference)
}
}
keptChunkRefs := make([]string, 0, len(chunkRefs))
for _, reference := range chunkRefs {
if !removedChunks[reference] {
keptChunkRefs = append(keptChunkRefs, reference)
}
}
if referenceBearing && len(keptSourceRefs) == 0 && len(keptChunkRefs) == 0 {
slug, _ := page["slug"].(string)
if slug != "" {
droppedSlugs[slug] = true
}
continue
}
page["source_refs"] = keptSourceRefs
page["chunk_refs"] = keptChunkRefs
if folderID, ok := page["folder_id"].(string); ok && folderID != "" {
keptFolderIDs[folderID] = true
}
pending = append(pending, page)
}
for _, page := range pending {
if slug, ok := page["parent_slug"].(string); ok && slug != "" && droppedSlugs[slug] {
page["parent_slug"] = ""
}
page["in_links"] = pruneSlugReferences(stringList(page["in_links"]), droppedSlugs)
page["out_links"] = pruneSlugReferences(stringList(page["out_links"]), droppedSlugs)
encoded, err := json.Marshal(page)
if err != nil {
return nil, nil, errors.New("rewrite retracted wiki page")
}
kept = append(kept, encoded)
}
return kept, pruneEmptyFolders(folders, keptFolderIDs), nil
}
// pruneEmptyFolders 只保留仍被页面引用的文件夹及其祖先链。
func pruneEmptyFolders(folders []json.RawMessage, referenced map[string]bool) []json.RawMessage {
type folderNode struct {
id string
parentID string
raw json.RawMessage
}
nodes := make(map[string]folderNode, len(folders))
for _, raw := range folders {
var row struct {
ID string `json:"id"`
ParentID string `json:"parent_id"`
}
if json.Unmarshal(raw, &row) != nil || row.ID == "" {
continue
}
nodes[row.ID] = folderNode{id: row.ID, parentID: row.ParentID, raw: raw}
}
keep := map[string]bool{}
for folderID := range referenced {
for current, guard := folderID, 0; current != "" && guard < len(nodes)+1; guard++ {
node, present := nodes[current]
if !present || keep[current] {
break
}
keep[current] = true
current = node.parentID
}
}
result := make([]json.RawMessage, 0, len(folders))
for _, raw := range folders {
var row struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &row) != nil || row.ID == "" {
result = append(result, raw)
continue
}
if keep[row.ID] {
result = append(result, raw)
}
}
return result
}
func decodeWikiRow(raw json.RawMessage) (map[string]any, error) {
var page map[string]any
if err := json.Unmarshal(raw, &page); err != nil {
return nil, errors.New("invalid wiki page row")
}
return page, nil
}
func stringList(value any) []string {
items, ok := value.([]any)
if !ok {
return nil
}
result := make([]string, 0, len(items))
for _, item := range items {
if text, ok := item.(string); ok {
result = append(result, text)
}
}
return result
}
func referencesKnowledge(reference string, knowledgeID string) bool {
return reference == knowledgeID || strings.HasPrefix(reference, knowledgeID+"|")
}
func pruneSlugReferences(slugs []string, dropped map[string]bool) []string {
result := make([]string, 0, len(slugs))
for _, slug := range slugs {
if !dropped[slug] {
result = append(result, slug)
}
}
return result
}
func matchesWikiPageRow(raw json.RawMessage, identity PostgresIdentity) bool {
var row struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Slug string `json:"slug"`
}
if json.Unmarshal(raw, &row) != nil {
return false
}
return row.ID != "" && row.Slug != "" && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID
}
func matchesWikiFolderRow(raw json.RawMessage, identity PostgresIdentity) bool {
var row struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Name string `json:"name"`
}
if json.Unmarshal(raw, &row) != nil {
return false
}
return row.ID != "" && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID
}
// hasWikiSnapshot 判断一个 bundle 是否携带库级 wiki 快照。老包(白名单里没有
// 这两张表)不参与 wiki 同步,生产 wiki 保持原样。
func hasWikiSnapshot(snapshot PostgresSnapshot) bool {
return tablePresent(snapshot, "wiki_pages") && tablePresent(snapshot, "wiki_folders")
}
func tablePresent(snapshot PostgresSnapshot, name string) bool {
for _, table := range snapshot.Tables {
if table.Name == name {
return true
}
}
return false
}
// wikiSnapshotRows 返回快照里的 wiki 行(供导入端做库级 replace 使用)。
func wikiSnapshotRows(snapshot PostgresSnapshot) ([]TableSnapshot, bool) {
if !hasWikiSnapshot(snapshot) {
return nil, false
}
return []TableSnapshot{snapshot.Table("wiki_folders"), snapshot.Table("wiki_pages")}, true
}