dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
286 lines
8.7 KiB
Go
286 lines
8.7 KiB
Go
package spool
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bufio"
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"hash"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"knowledge-governance/internal/governance"
|
|
)
|
|
|
|
type Expectation struct {
|
|
RelativeRef string
|
|
SHA256 string
|
|
SizeBytes int64
|
|
Format string
|
|
}
|
|
|
|
type Artifact struct {
|
|
file *os.File
|
|
expectedSHA string
|
|
expectedSize int64
|
|
streamHash hash.Hash
|
|
streamed int64
|
|
}
|
|
|
|
var supportedFormats = map[string]struct{}{
|
|
"pdf": {}, "docx": {}, "doc": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {},
|
|
"md": {}, "txt": {}, "csv": {}, "json": {},
|
|
}
|
|
|
|
func InspectVerified(root, relativeRef, format string, maxBytes int64) (Expectation, error) {
|
|
if !filepath.IsAbs(root) || relativeRef == "" || filepath.IsAbs(relativeRef) || maxBytes <= 0 {
|
|
return Expectation{}, errors.New("invalid spool reference")
|
|
}
|
|
cleanRef := filepath.Clean(relativeRef)
|
|
if cleanRef == "." || cleanRef != relativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
|
|
return Expectation{}, errors.New("invalid spool reference")
|
|
}
|
|
if _, ok := supportedFormats[format]; !ok {
|
|
return Expectation{}, errors.New("unsupported spool artifact format")
|
|
}
|
|
file, err := openRegularNoSymlink(root, cleanRef)
|
|
if err != nil {
|
|
return Expectation{}, errors.New("open spool artifact")
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 || info.Size() > maxBytes {
|
|
return Expectation{}, errors.New("spool artifact size or type mismatch")
|
|
}
|
|
digest := sha256.New()
|
|
bytesRead, err := io.Copy(digest, file)
|
|
if err != nil || bytesRead != info.Size() {
|
|
return Expectation{}, errors.New("inspect spool artifact content")
|
|
}
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return Expectation{}, errors.New("rewind spool artifact")
|
|
}
|
|
if err := validateFormat(file, format, info.Size()); err != nil {
|
|
return Expectation{}, errors.New("spool artifact format mismatch")
|
|
}
|
|
return Expectation{
|
|
RelativeRef: cleanRef,
|
|
SHA256: hex.EncodeToString(digest.Sum(nil)),
|
|
SizeBytes: info.Size(),
|
|
Format: format,
|
|
}, nil
|
|
}
|
|
|
|
func OpenVerified(root string, expected Expectation, maxBytes int64) (*Artifact, error) {
|
|
if !filepath.IsAbs(root) || expected.RelativeRef == "" || filepath.IsAbs(expected.RelativeRef) {
|
|
return nil, errors.New("invalid spool reference")
|
|
}
|
|
cleanRef := filepath.Clean(expected.RelativeRef)
|
|
if cleanRef == "." || cleanRef != expected.RelativeRef || cleanRef == ".." || strings.HasPrefix(cleanRef, ".."+string(filepath.Separator)) {
|
|
return nil, errors.New("invalid spool reference")
|
|
}
|
|
if !governance.ValidSHA256(expected.SHA256) || expected.SizeBytes <= 0 || maxBytes <= 0 || expected.SizeBytes > maxBytes {
|
|
return nil, errors.New("invalid spool expectation")
|
|
}
|
|
if _, ok := supportedFormats[expected.Format]; !ok {
|
|
return nil, errors.New("unsupported spool artifact format")
|
|
}
|
|
file, err := openRegularNoSymlink(root, cleanRef)
|
|
if err != nil {
|
|
return nil, errors.New("open spool artifact")
|
|
}
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
file.Close()
|
|
return nil, errors.New("inspect spool artifact")
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() != expected.SizeBytes {
|
|
file.Close()
|
|
return nil, errors.New("spool artifact size or type mismatch")
|
|
}
|
|
beforeHash := sha256.New()
|
|
bytesRead, err := io.Copy(beforeHash, file)
|
|
if err != nil || bytesRead != expected.SizeBytes || hex.EncodeToString(beforeHash.Sum(nil)) != expected.SHA256 {
|
|
file.Close()
|
|
return nil, errors.New("spool artifact content mismatch")
|
|
}
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
file.Close()
|
|
return nil, errors.New("rewind spool artifact")
|
|
}
|
|
if err := validateFormat(file, expected.Format, expected.SizeBytes); err != nil {
|
|
file.Close()
|
|
return nil, errors.New("spool artifact format mismatch")
|
|
}
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
file.Close()
|
|
return nil, errors.New("rewind spool artifact after validation")
|
|
}
|
|
return &Artifact{
|
|
file: file,
|
|
expectedSHA: expected.SHA256,
|
|
expectedSize: expected.SizeBytes,
|
|
streamHash: sha256.New(),
|
|
}, nil
|
|
}
|
|
|
|
func validateFormat(file *os.File, format string, size int64) error {
|
|
switch format {
|
|
case "pdf":
|
|
header := make([]byte, len("%PDF-"))
|
|
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, []byte("%PDF-")) {
|
|
return errors.New("invalid PDF magic")
|
|
}
|
|
case "docx", "xlsx", "pptx":
|
|
archive, err := zip.NewReader(file, size)
|
|
if err != nil {
|
|
return errors.New("invalid Office ZIP")
|
|
}
|
|
required := map[string]string{
|
|
"docx": "word/document.xml",
|
|
"xlsx": "xl/workbook.xml",
|
|
"pptx": "ppt/presentation.xml",
|
|
}[format]
|
|
foundContentTypes := false
|
|
foundRequired := false
|
|
for _, entry := range archive.File {
|
|
switch entry.Name {
|
|
case "[Content_Types].xml":
|
|
foundContentTypes = true
|
|
case required:
|
|
foundRequired = true
|
|
}
|
|
}
|
|
if !foundContentTypes || !foundRequired {
|
|
return errors.New("missing Office package entry")
|
|
}
|
|
case "doc", "xls", "ppt":
|
|
header := make([]byte, 8)
|
|
oleMagic := []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
|
|
if _, err := io.ReadFull(file, header); err != nil || !bytes.Equal(header, oleMagic) {
|
|
return errors.New("invalid legacy Office magic")
|
|
}
|
|
case "json":
|
|
decoder := json.NewDecoder(file)
|
|
var value any
|
|
if err := decoder.Decode(&value); err != nil {
|
|
return errors.New("invalid JSON")
|
|
}
|
|
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
|
return errors.New("multiple JSON values")
|
|
}
|
|
case "md", "txt", "csv":
|
|
prefixSize := size
|
|
if prefixSize > 4096 {
|
|
prefixSize = 4096
|
|
}
|
|
prefix := make([]byte, prefixSize)
|
|
if _, err := io.ReadFull(file, prefix); err != nil {
|
|
return errors.New("read text prefix")
|
|
}
|
|
lowerPrefix := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(string(prefix), "\ufeff")))
|
|
if strings.HasPrefix(lowerPrefix, "<!doctype html") || strings.HasPrefix(lowerPrefix, "<html") {
|
|
return errors.New("HTML is not a supported text artifact")
|
|
}
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return errors.New("rewind text artifact")
|
|
}
|
|
reader := bufio.NewReader(file)
|
|
for {
|
|
character, width, err := reader.ReadRune()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil || character == 0 || (character == utf8.RuneError && width == 1) {
|
|
return errors.New("invalid text content")
|
|
}
|
|
}
|
|
default:
|
|
return errors.New("unsupported format")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func openRegularNoSymlink(rootPath, relativeRef string) (*os.File, error) {
|
|
rootInfo, err := os.Lstat(rootPath)
|
|
if err != nil || rootInfo.Mode()&os.ModeSymlink != 0 || !rootInfo.IsDir() {
|
|
return nil, errors.New("invalid spool root")
|
|
}
|
|
current, err := os.OpenRoot(rootPath)
|
|
if err != nil {
|
|
return nil, errors.New("open spool root")
|
|
}
|
|
openedRootInfo, err := current.Stat(".")
|
|
if err != nil || !os.SameFile(rootInfo, openedRootInfo) {
|
|
current.Close()
|
|
return nil, errors.New("spool root changed while opening")
|
|
}
|
|
|
|
parts := strings.Split(relativeRef, string(filepath.Separator))
|
|
for _, part := range parts[:len(parts)-1] {
|
|
before, err := current.Lstat(part)
|
|
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.IsDir() {
|
|
current.Close()
|
|
return nil, errors.New("invalid spool directory component")
|
|
}
|
|
next, err := current.OpenRoot(part)
|
|
if err != nil {
|
|
current.Close()
|
|
return nil, errors.New("open spool directory component")
|
|
}
|
|
after, err := next.Stat(".")
|
|
if err != nil || !os.SameFile(before, after) {
|
|
next.Close()
|
|
current.Close()
|
|
return nil, errors.New("spool directory changed while opening")
|
|
}
|
|
current.Close()
|
|
current = next
|
|
}
|
|
|
|
fileName := parts[len(parts)-1]
|
|
before, err := current.Lstat(fileName)
|
|
if err != nil || before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() {
|
|
current.Close()
|
|
return nil, errors.New("invalid spool artifact type")
|
|
}
|
|
file, err := current.Open(fileName)
|
|
if err != nil {
|
|
current.Close()
|
|
return nil, errors.New("open spool artifact")
|
|
}
|
|
after, err := file.Stat()
|
|
current.Close()
|
|
if err != nil || !os.SameFile(before, after) || !after.Mode().IsRegular() {
|
|
file.Close()
|
|
return nil, errors.New("spool artifact changed while opening")
|
|
}
|
|
return file, nil
|
|
}
|
|
|
|
func (artifact *Artifact) Read(buffer []byte) (int, error) {
|
|
count, err := artifact.file.Read(buffer)
|
|
if count > 0 {
|
|
artifact.streamed += int64(count)
|
|
_, _ = artifact.streamHash.Write(buffer[:count])
|
|
}
|
|
return count, err
|
|
}
|
|
|
|
func (artifact *Artifact) VerifyComplete() error {
|
|
if artifact.streamed != artifact.expectedSize || hex.EncodeToString(artifact.streamHash.Sum(nil)) != artifact.expectedSHA {
|
|
return errors.New("spool artifact changed during streaming")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (artifact *Artifact) Close() error {
|
|
return artifact.file.Close()
|
|
}
|