0b9dbe12cf
build / build (push) Successful in 1m13s
为什么:父审指出资源范围未关联外层 resources,会把别的库/租户资源算进来; Verify 只看行数与 ID,同 ID 改正文/改向量/损坏对象都能通过;集成测试还在用假对象存储。 - scopeWhere(resources)/resourcePaths 用 resource_bindings.resource_id=resources.id 精确关联, 覆盖 knowledge/chunk 绑定并限制租户;资源部件名与资源 ID 对齐。 - Verify 增加逐行内容摘要、索引文档摘要、对象字节 SHA256 校验。 - 预备份一致性改为内容指纹(数量+摘要),同数量改写也会失败。 - 新增真实 MinIO 对象存储的本机验收用例,覆盖隔离/损坏/独立恢复。
153 lines
6.0 KiB
Go
153 lines
6.0 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
)
|
|
|
|
type S3ObjectStoreConfig struct {
|
|
Provider string
|
|
Endpoint string
|
|
Region string
|
|
AccessKeyID string
|
|
SecretAccessKey string
|
|
Bucket string
|
|
Secure bool
|
|
ForcePathStyle bool
|
|
MaxObjectBytes int64
|
|
HTTPClient *http.Client
|
|
}
|
|
|
|
type S3ObjectStore struct {
|
|
provider string
|
|
bucket string
|
|
maxObjectBytes int64
|
|
client *s3.Client
|
|
}
|
|
|
|
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
|
|
config.Provider = strings.TrimSpace(config.Provider)
|
|
config.Endpoint = strings.TrimSpace(config.Endpoint)
|
|
config.Region = strings.TrimSpace(config.Region)
|
|
config.Bucket = strings.TrimSpace(config.Bucket)
|
|
if config.Region == "" {
|
|
config.Region = "us-east-1"
|
|
}
|
|
if config.MaxObjectBytes == 0 {
|
|
config.MaxObjectBytes = 100 * 1024 * 1024
|
|
}
|
|
endpoint, err := url.Parse(config.Endpoint)
|
|
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
|
|
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
|
|
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
|
|
return nil, errors.New("invalid S3 object store configuration")
|
|
}
|
|
httpClient := &http.Client{Timeout: 2 * time.Minute}
|
|
if config.HTTPClient != nil {
|
|
clone := *config.HTTPClient
|
|
httpClient = &clone
|
|
}
|
|
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
|
|
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
|
|
client := s3.New(s3.Options{
|
|
Region: config.Region,
|
|
BaseEndpoint: aws.String(endpoint.String()),
|
|
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
|
|
UsePathStyle: forcePathStyle,
|
|
HTTPClient: httpClient,
|
|
})
|
|
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
|
|
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
|
|
return nil
|
|
}
|
|
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
|
|
return errors.New("ensure release object bucket")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
|
|
if provider != store.provider || !validObjectKey(physicalPath) {
|
|
return nil, errors.New("invalid release object source")
|
|
}
|
|
objectPath, ok := providerPathFromLocator(physicalPath)
|
|
if !ok {
|
|
return nil, errors.New("invalid release object source")
|
|
}
|
|
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil {
|
|
return nil, errors.New("read release source object")
|
|
}
|
|
return object.Body, nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
|
|
if provider != store.provider || !validObjectLocator(objectPath) || size < 0 || size > store.maxObjectBytes {
|
|
return errors.New("invalid release object write")
|
|
}
|
|
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
|
|
return errors.New("write release object")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
|
|
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
|
|
if provider != store.provider || !validObjectLocator(objectPath) {
|
|
return errors.New("invalid release object delete")
|
|
}
|
|
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}); err != nil {
|
|
return errors.New("delete release object")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
|
|
if provider != store.provider || !validObjectLocator(objectPath) {
|
|
return 0, "", errors.New("invalid release object read")
|
|
}
|
|
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
|
|
return 0, "", errors.New("inspect release object")
|
|
}
|
|
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
|
if err != nil {
|
|
return 0, "", errors.New("read release object")
|
|
}
|
|
defer object.Body.Close()
|
|
hash := sha256.New()
|
|
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
|
|
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
|
|
return 0, "", errors.New("verify release object")
|
|
}
|
|
return written, hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|
|
|
|
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
|
|
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
|
|
return errors.New("invalid release object copy")
|
|
}
|
|
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
|
|
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
|
|
return errors.New("copy release object")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var _ ObjectStore = (*S3ObjectStore)(nil)
|
|
var _ ObjectReader = (*S3ObjectStore)(nil)
|