feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,285 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type Expectation struct {
|
||||
RelativeRef string
|
||||
SHA256 string
|
||||
SizeBytes int64
|
||||
Format string
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
file *os.File
|
||||
expectedSHA string
|
||||
expectedSize int64
|
||||
streamHash hash.Hash
|
||||
streamed int64
|
||||
}
|
||||
|
||||
var supportedFormats = map[string]struct{}{
|
||||
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {},
|
||||
"md": {}, "txt": {}, "csv": {}, "json": {},
|
||||
}
|
||||
|
||||
func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) {
|
||||
if !filepath.IsAbs(root) || relativeRef == "" || filepath.IsAbs(relativeRef) || maxBytes <= 0 {
|
||||
return Expectation{}, errors.New("invalid spool reference")
|
||||
}
|
||||
cleanRef := filepath.Clean(relativeRef)
|
||||
if cleanRef == "." || cleanRef != relativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
|
||||
return Expectation{}, errors.New("invalid spool reference")
|
||||
}
|
||||
if _, ok := supportedFormats[format]; !ok {
|
||||
return Expectation{}, errors.New("unsupported spool artifact format")
|
||||
}
|
||||
file, err := openRegularNoSymlink(root, cleanRef)
|
||||
if err != nil {
|
||||
return Expectation{}, errors.New("open spool artifact")
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxBytes {
|
||||
return Expectation{}, errors.New("spool artifact size or type mismatch")
|
||||
}
|
||||
digest := sha256.New()
|
||||
bytesRead, err := io.Copy(digest, file)
|
||||
if err != nil || bytesRead != info.Size() {
|
||||
return Expectation{}, errors.New("inspect spool artifact content")
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return Expectation{}, errors.New("rewind spool artifact")
|
||||
}
|
||||
if err := validateFormat(file, format, info.Size()); err != nil {
|
||||
return Expectation{}, errors.New("spool artifact format mismatch")
|
||||
}
|
||||
return Expectation{
|
||||
RelativeRef: cleanRef,
|
||||
SHA256: hex.EncodeToString(digest.Sum(nil)),
|
||||
SizeBytes: info.Size(),
|
||||
Format: format,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func OpenVerified(root string, expected Expectation, maxBytes int64) (*Artifact, error) {
|
||||
if !filepath.IsAbs(root) || expected.RelativeRef == "" || filepath.IsAbs(expected.RelativeRef) {
|
||||
return nil, errors.New("invalid spool reference")
|
||||
}
|
||||
cleanRef := filepath.Clean(expected.RelativeRef)
|
||||
if cleanRef == "." || cleanRef != expected.RelativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
|
||||
return nil, errors.New("invalid spool reference")
|
||||
}
|
||||
if !governance.ValidSHA256(expected.SHA256) || expected.SizeBytes <= 0 || maxBytes <= 0 || expected.SizeBytes > maxBytes {
|
||||
return nil, errors.New("invalid spool expectation")
|
||||
}
|
||||
if _, ok := supportedFormats[expected.Format]; !ok {
|
||||
return nil, errors.New("unsupported spool artifact format")
|
||||
}
|
||||
file, err := openRegularNoSymlink(root, cleanRef)
|
||||
if err != nil {
|
||||
return nil, errors.New("open spool artifact")
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, errors.New("inspect spool artifact")
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Size() != expected.SizeBytes {
|
||||
file.Close()
|
||||
return nil, errors.New("spool artifact size or type mismatch")
|
||||
}
|
||||
beforeHash := sha256.New()
|
||||
bytesRead, err := io.Copy(beforeHash, file)
|
||||
if err != nil || bytesRead != expected.SizeBytes || hex.EncodeToString(beforeHash.Sum(nil)) != expected.SHA256 {
|
||||
file.Close()
|
||||
return nil, errors.New("spool artifact content mismatch")
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
file.Close()
|
||||
return nil, errors.New("rewind spool artifact")
|
||||
}
|
||||
if err := validateFormat(file, expected.Format, expected.SizeBytes); err != nil {
|
||||
file.Close()
|
||||
return nil, errors.New("spool artifact format mismatch")
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
file.Close()
|
||||
return nil, errors.New("rewind spool artifact after validation")
|
||||
}
|
||||
return &Artifact{
|
||||
file: file,
|
||||
expectedSHA: expected.SHA256,
|
||||
expectedSize: expected.SizeBytes,
|
||||
streamHash: sha256.New(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateFormat(file *os.File, format string, size int64) error {
|
||||
switch format {
|
||||
case "pdf":
|
||||
header := make([]byte, len("%PDF-"))
|
||||
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) {
|
||||
return errors.New("invalid PDF magic")
|
||||
}
|
||||
case "docx", "xlsx", "pptx":
|
||||
archive, err := zip.NewReader(file, size)
|
||||
if err != nil {
|
||||
return errors.New("invalid Office ZIP")
|
||||
}
|
||||
required := map[string]string{
|
||||
"docx": "word/document.xml",
|
||||
"xlsx": "xl/workbook.xml",
|
||||
"pptx": "ppt/presentation.xml",
|
||||
}[format]
|
||||
foundContentTypes := false
|
||||
foundRequired := false
|
||||
for _, entry := range archive.File {
|
||||
switch entry.Name {
|
||||
case "[Content_Types].xml":
|
||||
foundContentTypes = true
|
||||
case required:
|
||||
foundRequired = true
|
||||
}
|
||||
}
|
||||
if !foundContentTypes || !foundRequired {
|
||||
return errors.New("missing Office package entry")
|
||||
}
|
||||
case "doc", "xls", "ppt":
|
||||
header := make([]byte, 8)
|
||||
oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
|
||||
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) {
|
||||
return errors.New("invalid legacy Office magic")
|
||||
}
|
||||
case "json":
|
||||
decoder := json.NewDecoder(file)
|
||||
var value any
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return errors.New("invalid JSON")
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
case "md", "txt", "csv":
|
||||
prefixSize := size
|
||||
if prefixSize > 4096 {
|
||||
prefixSize = 4096
|
||||
}
|
||||
prefix := make([]byte, prefixSize)
|
||||
if _, err := io.ReadFull(file, prefix); err != nil {
|
||||
return errors.New("read text prefix")
|
||||
}
|
||||
lowerPrefix := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(string(prefix), "\ufeff")))
|
||||
if strings.HasPrefix(lowerPrefix, "<!doctype html") || strings.HasPrefix(lowerPrefix, "<html") {
|
||||
return errors.New("HTML is not a supported text artifact")
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return errors.New("rewind text artifact")
|
||||
}
|
||||
reader := bufio.NewReader(file)
|
||||
for {
|
||||
character, width, err := reader.ReadRune()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil || character == 0 || (character == utf8.RuneError && width == 1) {
|
||||
return errors.New("invalid text content")
|
||||
}
|
||||
}
|
||||
default:
|
||||
return errors.New("unsupported format")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openRegularNoSymlink(rootPath, relativeRef string) (*os.File, error) {
|
||||
rootInfo, err := os.Lstat(rootPath)
|
||||
if err != nil || rootInfo.Mode()&os.ModeSymlink != 0 || !rootInfo.IsDir() {
|
||||
return nil, errors.New("invalid spool root")
|
||||
}
|
||||
current, err := os.OpenRoot(rootPath)
|
||||
if err != nil {
|
||||
return nil, errors.New("open spool root")
|
||||
}
|
||||
openedRootInfo, err := current.Stat(".")
|
||||
if err != nil || !os.SameFile(rootInfo, openedRootInfo) {
|
||||
current.Close()
|
||||
return nil, errors.New("spool root changed while opening")
|
||||
}
|
||||
|
||||
parts := strings.Split(relativeRef, string(filepath.Separator))
|
||||
for _, part := range parts[:len(parts)-1] {
|
||||
before, err := current.Lstat(part)
|
||||
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.IsDir() {
|
||||
current.Close()
|
||||
return nil, errors.New("invalid spool directory component")
|
||||
}
|
||||
next, err := current.OpenRoot(part)
|
||||
if err != nil {
|
||||
current.Close()
|
||||
return nil, errors.New("open spool directory component")
|
||||
}
|
||||
after, err := next.Stat(".")
|
||||
if err != nil || !os.SameFile(before, after) {
|
||||
next.Close()
|
||||
current.Close()
|
||||
return nil, errors.New("spool directory changed while opening")
|
||||
}
|
||||
current.Close()
|
||||
current = next
|
||||
}
|
||||
|
||||
fileName := parts[len(parts)-1]
|
||||
before, err := current.Lstat(fileName)
|
||||
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
|
||||
current.Close()
|
||||
return nil, errors.New("invalid spool artifact type")
|
||||
}
|
||||
file, err := current.Open(fileName)
|
||||
if err != nil {
|
||||
current.Close()
|
||||
return nil, errors.New("open spool artifact")
|
||||
}
|
||||
after, err := file.Stat()
|
||||
current.Close()
|
||||
if err != nil || !os.SameFile(before, after) || !after.Mode().IsRegular() {
|
||||
file.Close()
|
||||
return nil, errors.New("spool artifact changed while opening")
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (artifact *Artifact) Read(buffer []byte) (int, error) {
|
||||
count, err := artifact.file.Read(buffer)
|
||||
if count > 0 {
|
||||
artifact.streamed += int64(count)
|
||||
_, _ = artifact.streamHash.Write(buffer[:count])
|
||||
}
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (artifact *Artifact) VerifyComplete() error {
|
||||
if artifact.streamed != artifact.expectedSize || hex.EncodeToString(artifact.streamHash.Sum(nil)) != artifact.expectedSHA {
|
||||
return errors.New("spool artifact changed during streaming")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (artifact *Artifact) Close() error {
|
||||
return artifact.file.Close()
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package spool
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenVerifiedRechecksSizeAndSHAWhileStreaming(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "objects"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("safe governance document\n")
|
||||
ref := filepath.Join("objects", "artifact.md")
|
||||
if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
artifact, err := OpenVerified(root, Expectation{
|
||||
RelativeRef: ref,
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: int64(len(body)),
|
||||
Format: "md",
|
||||
}, 50*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenVerified() error = %v", err)
|
||||
}
|
||||
defer artifact.Close()
|
||||
streamed, err := io.ReadAll(artifact)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
if string(streamed) != string(body) {
|
||||
t.Fatalf("streamed body = %q", streamed)
|
||||
}
|
||||
if err := artifact.VerifyComplete(); err != nil {
|
||||
t.Fatalf("VerifyComplete() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectVerifiedReturnsCanonicalExpectation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
body := []byte("governance p3 unique fact\n")
|
||||
if err := os.Mkdir(filepath.Join(root, "manual"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ref := filepath.Join("manual", "fixture.md")
|
||||
if err := os.WriteFile(filepath.Join(root, ref), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
|
||||
got, err := InspectVerified(root, ref, "md", 1024)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectVerified() error = %v", err)
|
||||
}
|
||||
if got.RelativeRef != ref || got.SizeBytes != int64(len(body)) || got.SHA256 != hex.EncodeToString(digest[:]) || got.Format != "md" {
|
||||
t.Fatalf("expectation = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenVerifiedRejectsSymlinksAndPathEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
body := []byte("safe governance document\n")
|
||||
digest := sha256.Sum256(body)
|
||||
expected := Expectation{
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: int64(len(body)),
|
||||
Format: "md",
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(root, "real"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "real", "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join("real", "artifact.md"), filepath.Join(root, "file-link.md")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("real", filepath.Join(root, "dir-link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parent := filepath.Dir(root)
|
||||
rootLink := filepath.Join(parent, filepath.Base(root)+"-link")
|
||||
if err := os.Symlink(root, rootLink); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(rootLink)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
ref string
|
||||
}{
|
||||
{name: "absolute reference", root: root, ref: filepath.Join(root, "real", "artifact.md")},
|
||||
{name: "parent escape", root: root, ref: "../artifact.md"},
|
||||
{name: "file symlink", root: root, ref: "file-link.md"},
|
||||
{name: "directory symlink", root: root, ref: filepath.Join("dir-link", "artifact.md")},
|
||||
{name: "root symlink", root: rootLink, ref: filepath.Join("real", "artifact.md")},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expectation := expected
|
||||
expectation.RelativeRef = tt.ref
|
||||
artifact, err := OpenVerified(tt.root, expectation, 50*1024*1024)
|
||||
if artifact != nil {
|
||||
artifact.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("OpenVerified(%q) error = nil", tt.ref)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenVerifiedRejectsUnsupportedFormat(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
body := []byte("safe governance document\n")
|
||||
path := filepath.Join(root, "artifact.bin")
|
||||
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
artifact, err := OpenVerified(root, Expectation{
|
||||
RelativeRef: "artifact.bin",
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: int64(len(body)),
|
||||
Format: "exe",
|
||||
}, 50*1024*1024)
|
||||
if artifact != nil {
|
||||
artifact.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("OpenVerified() error = nil for unsupported format")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenVerifiedRejectsContentThatDoesNotMatchFormat(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
format string
|
||||
body []byte
|
||||
}{
|
||||
{name: "PDF magic", format: "pdf", body: []byte("not a PDF")},
|
||||
{name: "DOCX container", format: "docx", body: []byte("not a ZIP")},
|
||||
{name: "legacy Office magic", format: "doc", body: []byte("not OLE")},
|
||||
{name: "JSON syntax", format: "json", body: []byte("{")},
|
||||
{name: "text binary", format: "md", body: []byte{'a', 0, 'b'}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "artifact"), tt.body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(tt.body)
|
||||
artifact, err := OpenVerified(root, Expectation{
|
||||
RelativeRef: "artifact",
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: int64(len(tt.body)),
|
||||
Format: tt.format,
|
||||
}, 50*1024*1024)
|
||||
if artifact != nil {
|
||||
artifact.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("OpenVerified() error = nil for format %s", tt.format)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenVerifiedRejectsHTMLMasqueradingAsText(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
body := []byte(" <!doctype html><html><title>Sign in</title></html>")
|
||||
if err := os.WriteFile(filepath.Join(root, "artifact.md"), body, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
artifact, err := OpenVerified(root, Expectation{
|
||||
RelativeRef: "artifact.md",
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: int64(len(body)),
|
||||
Format: "md",
|
||||
}, 50*1024*1024)
|
||||
if artifact != nil {
|
||||
artifact.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("OpenVerified() error = nil for HTML login page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenVerifiedRejectsEmptyArtifact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "empty.md"), nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sha256.Sum256(nil)
|
||||
artifact, err := OpenVerified(root, Expectation{
|
||||
RelativeRef: "empty.md",
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
SizeBytes: 0,
|
||||
Format: "md",
|
||||
}, 50*1024*1024)
|
||||
if artifact != nil {
|
||||
artifact.Close()
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("OpenVerified() error = nil for empty artifact")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user