Files
knowledge-governance/internal/releaseadapter/v071/s3_object_store.go
T
姚章浩 0b9dbe12cf
build / build (push) Successful in 1m13s
fix(library): 资源范围精确关联 + 内容级校验 + 真实对象存储验收
为什么:父审指出资源范围未关联外层 resources,会把别的库/租户资源算进来;
Verify 只看行数与 ID,同 ID 改正文/改向量/损坏对象都能通过;集成测试还在用假对象存储。

- scopeWhere(resources)/resourcePaths 用 resource_bindings.resource_id=resources.id 精确关联,
  覆盖 knowledge/chunk 绑定并限制租户;资源部件名与资源 ID 对齐。
- Verify 增加逐行内容摘要、索引文档摘要、对象字节 SHA256 校验。
- 预备份一致性改为内容指纹(数量+摘要),同数量改写也会失败。
- 新增真实 MinIO 对象存储的本机验收用例,覆盖隔离/损坏/独立恢复。
2026-10-28 16:23:28 +08:00

153 lines
6.0 KiB
Go

package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3ObjectStoreConfig struct {
Provider string
Endpoint string
Region string
AccessKeyID string
SecretAccessKey string
Bucket string
Secure bool
ForcePathStyle bool
MaxObjectBytes int64
HTTPClient *http.Client
}
type S3ObjectStore struct {
provider string
bucket string
maxObjectBytes int64
client *s3.Client
}
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
config.Provider = strings.TrimSpace(config.Provider)
config.Endpoint = strings.TrimSpace(config.Endpoint)
config.Region = strings.TrimSpace(config.Region)
config.Bucket = strings.TrimSpace(config.Bucket)
if config.Region == "" {
config.Region = "us-east-1"
}
if config.MaxObjectBytes == 0 {
config.MaxObjectBytes = 100 * 1024 * 1024
}
endpoint, err := url.Parse(config.Endpoint)
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
return nil, errors.New("invalid S3 object store configuration")
}
httpClient := &http.Client{Timeout: 2 * time.Minute}
if config.HTTPClient != nil {
clone := *config.HTTPClient
httpClient = &clone
}
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
client := s3.New(s3.Options{
Region: config.Region,
BaseEndpoint: aws.String(endpoint.String()),
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
UsePathStyle: forcePathStyle,
HTTPClient: httpClient,
})
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
}
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
return nil
}
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
return errors.New("ensure release object bucket")
}
return nil
}
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
if provider != store.provider || !validObjectKey(physicalPath) {
return nil, errors.New("invalid release object source")
}
objectPath, ok := providerPathFromLocator(physicalPath)
if !ok {
return nil, errors.New("invalid release object source")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return nil, errors.New("read release source object")
}
return object.Body, nil
}
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
if provider != store.provider || !validObjectLocator(objectPath) || size < 0 || size > store.maxObjectBytes {
return errors.New("invalid release object write")
}
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
return errors.New("write release object")
}
return nil
}
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
if provider != store.provider || !validObjectLocator(objectPath) {
return errors.New("invalid release object delete")
}
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}); err != nil {
return errors.New("delete release object")
}
return nil
}
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
if provider != store.provider || !validObjectLocator(objectPath) {
return 0, "", errors.New("invalid release object read")
}
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
return 0, "", errors.New("inspect release object")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return 0, "", errors.New("read release object")
}
defer object.Body.Close()
hash := sha256.New()
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
return 0, "", errors.New("verify release object")
}
return written, hex.EncodeToString(hash.Sum(nil)), nil
}
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
return errors.New("invalid release object copy")
}
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
return errors.New("copy release object")
}
return nil
}
var _ ObjectStore = (*S3ObjectStore)(nil)
var _ ObjectReader = (*S3ObjectStore)(nil)