Files
knowledge-governance/internal/platform/spool/artifact.go
T
zheromiku dcdc8d1ddd feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
2026-09-30 10:54:06 +08:00

286 lines
8.7 KiB
Go

package spool
import (
"archive/zip"
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"hash"
"io"
"os"
"path/filepath"
"strings"
"unicode/utf8"
"knowledge-governance/internal/governance"
)
type Expectation struct {
RelativeRef string
SHA256 string
SizeBytes int64
Format string
}
type Artifact struct {
file *os.File
expectedSHA string
expectedSize int64
streamHash hash.Hash
streamed int64
}
var supportedFormats = map[string]struct{}{
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {},
"md": {}, "txt": {}, "csv": {}, "json": {},
}
func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) {
if !filepath.IsAbs(root) || relativeRef == "" || filepath.IsAbs(relativeRef) || maxBytes <= 0 {
return Expectation{}, errors.New("invalid spool reference")
}
cleanRef := filepath.Clean(relativeRef)
if cleanRef == "." || cleanRef != relativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
return Expectation{}, errors.New("invalid spool reference")
}
if _, ok := supportedFormats[format]; !ok {
return Expectation{}, errors.New("unsupported spool artifact format")
}
file, err := openRegularNoSymlink(root, cleanRef)
if err != nil {
return Expectation{}, errors.New("open spool artifact")
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxBytes {
return Expectation{}, errors.New("spool artifact size or type mismatch")
}
digest := sha256.New()
bytesRead, err := io.Copy(digest, file)
if err != nil || bytesRead != info.Size() {
return Expectation{}, errors.New("inspect spool artifact content")
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return Expectation{}, errors.New("rewind spool artifact")
}
if err := validateFormat(file, format, info.Size()); err != nil {
return Expectation{}, errors.New("spool artifact format mismatch")
}
return Expectation{
RelativeRef: cleanRef,
SHA256: hex.EncodeToString(digest.Sum(nil)),
SizeBytes: info.Size(),
Format: format,
}, nil
}
func OpenVerified(root string, expected Expectation, maxBytes int64) (*Artifact, error) {
if !filepath.IsAbs(root) || expected.RelativeRef == "" || filepath.IsAbs(expected.RelativeRef) {
return nil, errors.New("invalid spool reference")
}
cleanRef := filepath.Clean(expected.RelativeRef)
if cleanRef == "." || cleanRef != expected.RelativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
return nil, errors.New("invalid spool reference")
}
if !governance.ValidSHA256(expected.SHA256) || expected.SizeBytes <= 0 || maxBytes <= 0 || expected.SizeBytes > maxBytes {
return nil, errors.New("invalid spool expectation")
}
if _, ok := supportedFormats[expected.Format]; !ok {
return nil, errors.New("unsupported spool artifact format")
}
file, err := openRegularNoSymlink(root, cleanRef)
if err != nil {
return nil, errors.New("open spool artifact")
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, errors.New("inspect spool artifact")
}
if !info.Mode().IsRegular() || info.Size() != expected.SizeBytes {
file.Close()
return nil, errors.New("spool artifact size or type mismatch")
}
beforeHash := sha256.New()
bytesRead, err := io.Copy(beforeHash, file)
if err != nil || bytesRead != expected.SizeBytes || hex.EncodeToString(beforeHash.Sum(nil)) != expected.SHA256 {
file.Close()
return nil, errors.New("spool artifact content mismatch")
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
file.Close()
return nil, errors.New("rewind spool artifact")
}
if err := validateFormat(file, expected.Format, expected.SizeBytes); err != nil {
file.Close()
return nil, errors.New("spool artifact format mismatch")
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
file.Close()
return nil, errors.New("rewind spool artifact after validation")
}
return &Artifact{
file: file,
expectedSHA: expected.SHA256,
expectedSize: expected.SizeBytes,
streamHash: sha256.New(),
}, nil
}
func validateFormat(file *os.File, format string, size int64) error {
switch format {
case "pdf":
header := make([]byte, len("%PDF-"))
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) {
return errors.New("invalid PDF magic")
}
case "docx", "xlsx", "pptx":
archive, err := zip.NewReader(file, size)
if err != nil {
return errors.New("invalid Office ZIP")
}
required := map[string]string{
"docx": "word/document.xml",
"xlsx": "xl/workbook.xml",
"pptx": "ppt/presentation.xml",
}[format]
foundContentTypes := false
foundRequired := false
for _, entry := range archive.File {
switch entry.Name {
case "[Content_Types].xml":
foundContentTypes = true
case required:
foundRequired = true
}
}
if !foundContentTypes || !foundRequired {
return errors.New("missing Office package entry")
}
case "doc", "xls", "ppt":
header := make([]byte, 8)
oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) {
return errors.New("invalid legacy Office magic")
}
case "json":
decoder := json.NewDecoder(file)
var value any
if err := decoder.Decode(&value); err != nil {
return errors.New("invalid JSON")
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return errors.New("multiple JSON values")
}
case "md", "txt", "csv":
prefixSize := size
if prefixSize > 4096 {
prefixSize = 4096
}
prefix := make([]byte, prefixSize)
if _, err := io.ReadFull(file, prefix); err != nil {
return errors.New("read text prefix")
}
lowerPrefix := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(string(prefix), "\ufeff")))
if strings.HasPrefix(lowerPrefix, "<!doctype html") || strings.HasPrefix(lowerPrefix, "<html") {
return errors.New("HTML is not a supported text artifact")
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return errors.New("rewind text artifact")
}
reader := bufio.NewReader(file)
for {
character, width, err := reader.ReadRune()
if err == io.EOF {
break
}
if err != nil || character == 0 || (character == utf8.RuneError && width == 1) {
return errors.New("invalid text content")
}
}
default:
return errors.New("unsupported format")
}
return nil
}
func openRegularNoSymlink(rootPath, relativeRef string) (*os.File, error) {
rootInfo, err := os.Lstat(rootPath)
if err != nil || rootInfo.Mode()&os.ModeSymlink != 0 || !rootInfo.IsDir() {
return nil, errors.New("invalid spool root")
}
current, err := os.OpenRoot(rootPath)
if err != nil {
return nil, errors.New("open spool root")
}
openedRootInfo, err := current.Stat(".")
if err != nil || !os.SameFile(rootInfo, openedRootInfo) {
current.Close()
return nil, errors.New("spool root changed while opening")
}
parts := strings.Split(relativeRef, string(filepath.Separator))
for _, part := range parts[:len(parts)-1] {
before, err := current.Lstat(part)
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.IsDir() {
current.Close()
return nil, errors.New("invalid spool directory component")
}
next, err := current.OpenRoot(part)
if err != nil {
current.Close()
return nil, errors.New("open spool directory component")
}
after, err := next.Stat(".")
if err != nil || !os.SameFile(before, after) {
next.Close()
current.Close()
return nil, errors.New("spool directory changed while opening")
}
current.Close()
current = next
}
fileName := parts[len(parts)-1]
before, err := current.Lstat(fileName)
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
current.Close()
return nil, errors.New("invalid spool artifact type")
}
file, err := current.Open(fileName)
if err != nil {
current.Close()
return nil, errors.New("open spool artifact")
}
after, err := file.Stat()
current.Close()
if err != nil || !os.SameFile(before, after) || !after.Mode().IsRegular() {
file.Close()
return nil, errors.New("spool artifact changed while opening")
}
return file, nil
}
func (artifact *Artifact) Read(buffer []byte) (int, error) {
count, err := artifact.file.Read(buffer)
if count > 0 {
artifact.streamed += int64(count)
_, _ = artifact.streamHash.Write(buffer[:count])
}
return count, err
}
func (artifact *Artifact) VerifyComplete() error {
if artifact.streamed != artifact.expectedSize || hex.EncodeToString(artifact.streamHash.Sum(nil)) != artifact.expectedSHA {
return errors.New("spool artifact changed during streaming")
}
return nil
}
func (artifact *Artifact) Close() error {
return artifact.file.Close()
}