dcdc8d1ddd
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
142 lines
5.5 KiB
Go
142 lines
5.5 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
)
|
|
|
|
type S3ObjectStoreConfig struct {
|
|
Provider string
|
|
Endpoint string
|
|
Region string
|
|
AccessKeyID string
|
|
SecretAccessKey string
|
|
Bucket string
|
|
Secure bool
|
|
ForcePathStyle bool
|
|
MaxObjectBytes int64
|
|
HTTPClient *http.Client
|
|
}
|
|
|
|
type S3ObjectStore struct {
|
|
provider string
|
|
bucket string
|
|
maxObjectBytes int64
|
|
client *s3.Client
|
|
}
|
|
|
|
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
|
|
config.Provider = strings.TrimSpace(config.Provider)
|
|
config.Endpoint = strings.TrimSpace(config.Endpoint)
|
|
config.Region = strings.TrimSpace(config.Region)
|
|
config.Bucket = strings.TrimSpace(config.Bucket)
|
|
if config.Region == "" {
|
|
config.Region = "us-east-1"
|
|
}
|
|
if config.MaxObjectBytes == 0 {
|
|
config.MaxObjectBytes = 100 * 1024 * 1024
|
|
}
|
|
endpoint, err := url.Parse(config.Endpoint)
|
|
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
|
|
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
|
|
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
|
|
return nil, errors.New("invalid S3 object store configuration")
|
|
}
|
|
httpClient := &http.Client{Timeout: 2 * time.Minute}
|
|
if config.HTTPClient != nil {
|
|
clone := *config.HTTPClient
|
|
httpClient = &clone
|
|
}
|
|
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
|
|
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
|
|
client := s3.New(s3.Options{
|
|
Region: config.Region,
|
|
BaseEndpoint: aws.String(endpoint.String()),
|
|
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
|
|
UsePathStyle: forcePathStyle,
|
|
HTTPClient: httpClient,
|
|
})
|
|
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
|
|
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
|
|
return nil
|
|
}
|
|
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
|
|
return errors.New("ensure release object bucket")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
|
|
if provider != store.provider || !validObjectKey(physicalPath) {
|
|
return nil, errors.New("invalid release object source")
|
|
}
|
|
objectPath, ok := providerPathFromLocator(physicalPath)
|
|
if !ok {
|
|
return nil, errors.New("invalid release object source")
|
|
}
|
|
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil {
|
|
return nil, errors.New("read release source object")
|
|
}
|
|
return object.Body, nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
|
|
if provider != store.provider || !validObjectLocator(objectPath) || size < 0 || size > store.maxObjectBytes {
|
|
return errors.New("invalid release object write")
|
|
}
|
|
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
|
|
return errors.New("write release object")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
|
|
if provider != store.provider || !validObjectLocator(objectPath) {
|
|
return 0, "", errors.New("invalid release object read")
|
|
}
|
|
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
|
|
return 0, "", errors.New("inspect release object")
|
|
}
|
|
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil {
|
|
return 0, "", errors.New("read release object")
|
|
}
|
|
defer object.Body.Close()
|
|
hash := sha256.New()
|
|
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
|
|
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
|
|
return 0, "", errors.New("verify release object")
|
|
}
|
|
return written, hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
|
|
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
|
|
return errors.New("invalid release object copy")
|
|
}
|
|
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
|
|
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
|
|
return errors.New("copy release object")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var _ ObjectStore = (*S3ObjectStore)(nil)
|
|
var _ ObjectReader = (*S3ObjectStore)(nil)
|