Files
knowledge-governance/internal/releaseadapter/v071/s3_object_store.go
T
zheromiku dcdc8d1ddd feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环):

- 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口)
- 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入
  parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截
- 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过):
  * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id)
  * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配
  * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数
  * 回滚 job payload 支持 prior_execution_id(strict 解码)
- 三通道发布+回滚演练实测:publish 4/4 verified,检索命中,
  rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false
- 前端 A·Data-console 风格全站统一,120 用例通过
- 门禁:go vet / go test 全包通过,gofmt 干净
2026-09-30 10:54:06 +08:00

142 lines
5.5 KiB
Go

package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3ObjectStoreConfig struct {
Provider string
Endpoint string
Region string
AccessKeyID string
SecretAccessKey string
Bucket string
Secure bool
ForcePathStyle bool
MaxObjectBytes int64
HTTPClient *http.Client
}
type S3ObjectStore struct {
provider string
bucket string
maxObjectBytes int64
client *s3.Client
}
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
config.Provider = strings.TrimSpace(config.Provider)
config.Endpoint = strings.TrimSpace(config.Endpoint)
config.Region = strings.TrimSpace(config.Region)
config.Bucket = strings.TrimSpace(config.Bucket)
if config.Region == "" {
config.Region = "us-east-1"
}
if config.MaxObjectBytes == 0 {
config.MaxObjectBytes = 100 * 1024 * 1024
}
endpoint, err := url.Parse(config.Endpoint)
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
return nil, errors.New("invalid S3 object store configuration")
}
httpClient := &http.Client{Timeout: 2 * time.Minute}
if config.HTTPClient != nil {
clone := *config.HTTPClient
httpClient = &clone
}
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
client := s3.New(s3.Options{
Region: config.Region,
BaseEndpoint: aws.String(endpoint.String()),
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
UsePathStyle: forcePathStyle,
HTTPClient: httpClient,
})
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
}
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
return nil
}
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
return errors.New("ensure release object bucket")
}
return nil
}
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
if provider != store.provider || !validObjectKey(physicalPath) {
return nil, errors.New("invalid release object source")
}
objectPath, ok := providerPathFromLocator(physicalPath)
if !ok {
return nil, errors.New("invalid release object source")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return nil, errors.New("read release source object")
}
return object.Body, nil
}
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
if provider != store.provider || !validObjectLocator(objectPath) || size < 0 || size > store.maxObjectBytes {
return errors.New("invalid release object write")
}
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
return errors.New("write release object")
}
return nil
}
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
if provider != store.provider || !validObjectLocator(objectPath) {
return 0, "", errors.New("invalid release object read")
}
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
return 0, "", errors.New("inspect release object")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return 0, "", errors.New("read release object")
}
defer object.Body.Close()
hash := sha256.New()
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
return 0, "", errors.New("verify release object")
}
return written, hex.EncodeToString(hash.Sum(nil)), nil
}
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
return errors.New("invalid release object copy")
}
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
return errors.New("copy release object")
}
return nil
}
var _ ObjectStore = (*S3ObjectStore)(nil)
var _ ObjectReader = (*S3ObjectStore)(nil)