ea6e56b304
发布通道原来只有三条:PostgreSQL 关系行、Elasticsearch 分片、OSS 对象。 WeKnora 的 wiki 图谱是 wiki_pages 一张纯 PG 表(in_links/out_links 存 slug), 所以它不在任何通道里——撤权后生产 wiki 会一直留着已撤权文档的页面与引用。 本次新增第 5 条通道:库级 wiki 快照,挂在现有文档发布包里,检索与 wiki 同一个 release 单元(同一次执行、同一次校验、同一次回滚): - 导出:ExportWiki 按 knowledge_base_id 全量读 wiki_pages/wiki_folders; 收敛门禁(库内无解析中/合成中文档)不满足直接失败并给中文原因; 未启用 wiki 且无 wiki 行的库不参与同步(生产 wiki 保持原样)。 - 撤权:item.Action == "delete" 时在导出端剥离该文档的 wiki 贡献 (source_refs/chunk_refs 去条目、无引用页面整页不带、摘死链、清 parent_slug、 无页面引用的空文件夹及其祖先一并去掉),使“源端仍在只停止发布”和 “源端删除”两种情况收敛到同一结果。 - 导入:库级 replace-by-kb(同一事务内 DELETE by kb 后灌入快照,文件夹先于页面), 新增 ReplaceWiki 供撤权路径使用(文档行只 disable,wiki 整体替换)。 - 校验:VerifyWiki 用生产 WeKnora wiki API 比对页面 slug 集合,撤权时额外断言 生产 wiki 不再引用被撤权文档。 驱动与验收:新增 cmd/release-channel-e2e(复用发布 worker 真实代码路径, 支持 publish/withdraw/rollback),scripts/wiki_e2e_test5.sh、 scripts/wiki_e2e_newkb.sh 两套端到端脚本;证据见 docs/WIKI_CO_TRANSPORT_ACCEPTANCE.md。
313 lines
12 KiB
Go
313 lines
12 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"strings"
|
||
)
|
||
|
||
// 库级 wiki 快照(第 5 条发布通道)。
|
||
//
|
||
// WeKnora 的 wiki 图谱不是 Neo4j 图谱:节点与边都在 wiki_pages 一张表里
|
||
// (in_links/out_links 存 slug 数组,库内相对引用),文件夹在 wiki_folders。
|
||
// 两者都是纯 PostgreSQL 数据,同一个 knowledge_base_id 可以直接搬运,
|
||
// 不需要重建,也不依赖 Neo4j / Elasticsearch。
|
||
//
|
||
// 快照粒度是“整库”,不是“单文档”:撤一篇文档会改动其它共贡页面的正文
|
||
// (retract 重写、摘死链、重建索引页),所以 delta 的最小单位只能是整库。
|
||
|
||
const exportWikiSettlementSQL = `
|
||
SELECT
|
||
(SELECT COUNT(*)
|
||
FROM knowledges AS knowledge
|
||
WHERE knowledge.tenant_id = $1 AND knowledge.knowledge_base_id = $2 AND knowledge.deleted_at IS NULL
|
||
AND (knowledge.pending_subtasks_count > 0 OR knowledge.parse_status IN ('pending', 'processing', 'finalizing'))),
|
||
(SELECT COUNT(*) FROM wiki_pages AS page WHERE page.tenant_id = $1 AND page.knowledge_base_id = $2),
|
||
(SELECT COUNT(*) FROM wiki_folders AS folder WHERE folder.tenant_id = $1 AND folder.knowledge_base_id = $2),
|
||
(SELECT COALESCE((base.indexing_strategy ->> 'wiki_enabled')::boolean, false)
|
||
FROM knowledge_bases AS base
|
||
WHERE base.id = $2 AND base.tenant_id = $1 AND base.deleted_at IS NULL)`
|
||
|
||
const exportWikiPagesSQL = `
|
||
SELECT to_jsonb(page)
|
||
FROM wiki_pages AS page
|
||
WHERE page.tenant_id = $1 AND page.knowledge_base_id = $2 AND page.deleted_at IS NULL
|
||
ORDER BY page.slug`
|
||
|
||
const exportWikiFoldersSQL = `
|
||
SELECT to_jsonb(folder)
|
||
FROM wiki_folders AS folder
|
||
WHERE folder.tenant_id = $1 AND folder.knowledge_base_id = $2 AND folder.deleted_at IS NULL
|
||
ORDER BY folder.id`
|
||
|
||
// wikiSnapshotExporter 由 PostgresExporter 实现;文档导出器在存在该能力时把
|
||
// 库级 wiki 快照追加进同一个 bundle,使检索与 wiki 落在同一个发布单元里。
|
||
type wikiSnapshotExporter interface {
|
||
ExportWiki(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string) ([]TableSnapshot, error)
|
||
}
|
||
|
||
// ExportWiki 导出该知识库的 wiki 快照。
|
||
//
|
||
// removingKnowledgeID 非空表示本次是撤权(删除)发布:剥掉该文档在 wiki 里的
|
||
// 贡献。源端“仅停用”不会改动 wiki(WeKnora 只在删除时跑 retract 清理链),
|
||
// 这一层剥离让“停止发布”和“源端删除”两种情况收敛到同一个结果:生产 wiki
|
||
// 不再引用该文档。源端真删除时清理链已经跑过,剥离是幂等空操作。
|
||
func (exporter *PostgresExporter) ExportWiki(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string) ([]TableSnapshot, error) {
|
||
if identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) {
|
||
return nil, errors.New("invalid wiki export identity")
|
||
}
|
||
if removingKnowledgeID != "" && !validSearchIdentifier(removingKnowledgeID) {
|
||
return nil, errors.New("invalid wiki removal identity")
|
||
}
|
||
var pending, pageCount, folderCount int
|
||
var wikiEnabled *bool
|
||
if err := exporter.db.QueryRow(ctx, exportWikiSettlementSQL, identity.TenantID, identity.KnowledgeBaseID).
|
||
Scan(&pending, &pageCount, &folderCount, &wikiEnabled); err != nil {
|
||
return nil, errors.New("read wiki snapshot state")
|
||
}
|
||
// 未启用 wiki 且库里没有任何 wiki 行的知识库不参与同步:包内不带 wiki 表,
|
||
// 生产端 wiki 保持原样,避免把生产 wiki 误清空。
|
||
if (wikiEnabled == nil || !*wikiEnabled) && pageCount == 0 && folderCount == 0 {
|
||
return nil, nil
|
||
}
|
||
// 收敛门禁:还有文档在解析或 wiki 合成中时不导出,避免产出半成品 wiki 快照。
|
||
if pending > 0 {
|
||
return nil, errors.New("源端 wiki 尚未收敛:该知识库仍有文档在解析或 wiki 合成中,请稍后重试")
|
||
}
|
||
pages, err := exporter.queryRows(ctx, exportWikiPagesSQL, identity.TenantID, identity.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
folders, err := exporter.queryRows(ctx, exportWikiFoldersSQL, identity.TenantID, identity.KnowledgeBaseID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, page := range pages {
|
||
if !matchesWikiPageRow(page, identity) {
|
||
return nil, errors.New("PostgreSQL returned a cross-scope wiki page")
|
||
}
|
||
}
|
||
for _, folder := range folders {
|
||
if !matchesWikiFolderRow(folder, identity) {
|
||
return nil, errors.New("PostgreSQL returned a cross-scope wiki folder")
|
||
}
|
||
}
|
||
if removingKnowledgeID != "" {
|
||
pages, folders, err = exporter.retractWikiContribution(ctx, identity, removingKnowledgeID, pages, folders)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
// wiki_folders 先于 wiki_pages:页面通过 folder_id 引用文件夹。
|
||
return []TableSnapshot{
|
||
newTableSnapshot("wiki_folders", folders),
|
||
newTableSnapshot("wiki_pages", pages),
|
||
}, nil
|
||
}
|
||
|
||
// retractWikiContribution 从库级快照里剥掉一篇文档的 wiki 贡献:
|
||
// - source_refs 形如 "<knowledge_id>|<doc_title>",剥掉该 knowledge 的条目;
|
||
// - chunk_refs 是该文档 chunk UUID,剥掉属于该 knowledge 的条目;
|
||
// - 剥完没有任何来源引用的页面整页不带走(等同源端“专属页删除”);
|
||
// 索引页 / 日志页本来就没有 source_refs,保留;
|
||
// - 指向被丢弃页面的 in_links / out_links 一并摘掉(避免生产死链);
|
||
// - 父页被丢弃时清空 parent_slug;
|
||
// - 没有页面再引用的文件夹(以及它们的空祖先)也一并去掉,避免生产 wiki
|
||
// 留下“空目录”这种撤权痕迹。
|
||
func (exporter *PostgresExporter) retractWikiContribution(ctx context.Context, identity PostgresIdentity, removingKnowledgeID string, pages, folders []json.RawMessage) ([]json.RawMessage, []json.RawMessage, error) {
|
||
chunks, err := exporter.queryRows(ctx, exportChunksSQL, removingKnowledgeID, identity.TenantID)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
removedChunks := make(map[string]bool, len(chunks))
|
||
for _, chunk := range chunks {
|
||
chunkID, ok := matchesChunkRow(chunk, identity)
|
||
if !ok {
|
||
return nil, nil, errors.New("PostgreSQL returned a cross-scope chunk row")
|
||
}
|
||
removedChunks[chunkID] = true
|
||
}
|
||
kept := make([]json.RawMessage, 0, len(pages))
|
||
droppedSlugs := map[string]bool{}
|
||
pending := make([]map[string]any, 0, len(pages))
|
||
keptFolderIDs := map[string]bool{}
|
||
for _, raw := range pages {
|
||
page, err := decodeWikiRow(raw)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
sourceRefs := stringList(page["source_refs"])
|
||
chunkRefs := stringList(page["chunk_refs"])
|
||
referenceBearing := len(sourceRefs) > 0 || len(chunkRefs) > 0
|
||
keptSourceRefs := make([]string, 0, len(sourceRefs))
|
||
for _, reference := range sourceRefs {
|
||
if !referencesKnowledge(reference, removingKnowledgeID) {
|
||
keptSourceRefs = append(keptSourceRefs, reference)
|
||
}
|
||
}
|
||
keptChunkRefs := make([]string, 0, len(chunkRefs))
|
||
for _, reference := range chunkRefs {
|
||
if !removedChunks[reference] {
|
||
keptChunkRefs = append(keptChunkRefs, reference)
|
||
}
|
||
}
|
||
if referenceBearing && len(keptSourceRefs) == 0 && len(keptChunkRefs) == 0 {
|
||
slug, _ := page["slug"].(string)
|
||
if slug != "" {
|
||
droppedSlugs[slug] = true
|
||
}
|
||
continue
|
||
}
|
||
page["source_refs"] = keptSourceRefs
|
||
page["chunk_refs"] = keptChunkRefs
|
||
if folderID, ok := page["folder_id"].(string); ok && folderID != "" {
|
||
keptFolderIDs[folderID] = true
|
||
}
|
||
pending = append(pending, page)
|
||
}
|
||
for _, page := range pending {
|
||
if slug, ok := page["parent_slug"].(string); ok && slug != "" && droppedSlugs[slug] {
|
||
page["parent_slug"] = ""
|
||
}
|
||
page["in_links"] = pruneSlugReferences(stringList(page["in_links"]), droppedSlugs)
|
||
page["out_links"] = pruneSlugReferences(stringList(page["out_links"]), droppedSlugs)
|
||
encoded, err := json.Marshal(page)
|
||
if err != nil {
|
||
return nil, nil, errors.New("rewrite retracted wiki page")
|
||
}
|
||
kept = append(kept, encoded)
|
||
}
|
||
return kept, pruneEmptyFolders(folders, keptFolderIDs), nil
|
||
}
|
||
|
||
// pruneEmptyFolders 只保留仍被页面引用的文件夹及其祖先链。
|
||
func pruneEmptyFolders(folders []json.RawMessage, referenced map[string]bool) []json.RawMessage {
|
||
type folderNode struct {
|
||
id string
|
||
parentID string
|
||
raw json.RawMessage
|
||
}
|
||
nodes := make(map[string]folderNode, len(folders))
|
||
for _, raw := range folders {
|
||
var row struct {
|
||
ID string `json:"id"`
|
||
ParentID string `json:"parent_id"`
|
||
}
|
||
if json.Unmarshal(raw, &row) != nil || row.ID == "" {
|
||
continue
|
||
}
|
||
nodes[row.ID] = folderNode{id: row.ID, parentID: row.ParentID, raw: raw}
|
||
}
|
||
keep := map[string]bool{}
|
||
for folderID := range referenced {
|
||
for current, guard := folderID, 0; current != "" && guard < len(nodes)+1; guard++ {
|
||
node, present := nodes[current]
|
||
if !present || keep[current] {
|
||
break
|
||
}
|
||
keep[current] = true
|
||
current = node.parentID
|
||
}
|
||
}
|
||
result := make([]json.RawMessage, 0, len(folders))
|
||
for _, raw := range folders {
|
||
var row struct {
|
||
ID string `json:"id"`
|
||
}
|
||
if json.Unmarshal(raw, &row) != nil || row.ID == "" {
|
||
result = append(result, raw)
|
||
continue
|
||
}
|
||
if keep[row.ID] {
|
||
result = append(result, raw)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func decodeWikiRow(raw json.RawMessage) (map[string]any, error) {
|
||
var page map[string]any
|
||
if err := json.Unmarshal(raw, &page); err != nil {
|
||
return nil, errors.New("invalid wiki page row")
|
||
}
|
||
return page, nil
|
||
}
|
||
|
||
func stringList(value any) []string {
|
||
items, ok := value.([]any)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
result := make([]string, 0, len(items))
|
||
for _, item := range items {
|
||
if text, ok := item.(string); ok {
|
||
result = append(result, text)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func referencesKnowledge(reference string, knowledgeID string) bool {
|
||
return reference == knowledgeID || strings.HasPrefix(reference, knowledgeID+"|")
|
||
}
|
||
|
||
func pruneSlugReferences(slugs []string, dropped map[string]bool) []string {
|
||
result := make([]string, 0, len(slugs))
|
||
for _, slug := range slugs {
|
||
if !dropped[slug] {
|
||
result = append(result, slug)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func matchesWikiPageRow(raw json.RawMessage, identity PostgresIdentity) bool {
|
||
var row struct {
|
||
ID string `json:"id"`
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
Slug string `json:"slug"`
|
||
}
|
||
if json.Unmarshal(raw, &row) != nil {
|
||
return false
|
||
}
|
||
return row.ID != "" && row.Slug != "" && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID
|
||
}
|
||
|
||
func matchesWikiFolderRow(raw json.RawMessage, identity PostgresIdentity) bool {
|
||
var row struct {
|
||
ID string `json:"id"`
|
||
TenantID uint64 `json:"tenant_id"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
Name string `json:"name"`
|
||
}
|
||
if json.Unmarshal(raw, &row) != nil {
|
||
return false
|
||
}
|
||
return row.ID != "" && row.TenantID == identity.TenantID && row.KnowledgeBaseID == identity.KnowledgeBaseID
|
||
}
|
||
|
||
// hasWikiSnapshot 判断一个 bundle 是否携带库级 wiki 快照。老包(白名单里没有
|
||
// 这两张表)不参与 wiki 同步,生产 wiki 保持原样。
|
||
func hasWikiSnapshot(snapshot PostgresSnapshot) bool {
|
||
return tablePresent(snapshot, "wiki_pages") && tablePresent(snapshot, "wiki_folders")
|
||
}
|
||
|
||
func tablePresent(snapshot PostgresSnapshot, name string) bool {
|
||
for _, table := range snapshot.Tables {
|
||
if table.Name == name {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// wikiSnapshotRows 返回快照里的 wiki 行(供导入端做库级 replace 使用)。
|
||
func wikiSnapshotRows(snapshot PostgresSnapshot) ([]TableSnapshot, bool) {
|
||
if !hasWikiSnapshot(snapshot) {
|
||
return nil, false
|
||
}
|
||
return []TableSnapshot{snapshot.Table("wiki_folders"), snapshot.Table("wiki_pages")}, true
|
||
}
|