ac3b395102
build / build (push) Successful in 1m21s
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设, 以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。 - knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。 - 写入顺序按真实外键;无代理键表用内容摘要核对。 - 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。 - 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。 - 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。 - finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。 - 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
160 lines
6.3 KiB
Go
160 lines
6.3 KiB
Go
package v071
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/aws/aws-sdk-go-v2/aws"
|
||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||
)
|
||
|
||
type S3ObjectStoreConfig struct {
|
||
Provider string
|
||
Endpoint string
|
||
Region string
|
||
AccessKeyID string
|
||
SecretAccessKey string
|
||
Bucket string
|
||
Secure bool
|
||
ForcePathStyle bool
|
||
MaxObjectBytes int64
|
||
HTTPClient *http.Client
|
||
}
|
||
|
||
type S3ObjectStore struct {
|
||
provider string
|
||
bucket string
|
||
maxObjectBytes int64
|
||
client *s3.Client
|
||
}
|
||
|
||
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
|
||
config.Provider = strings.TrimSpace(config.Provider)
|
||
config.Endpoint = strings.TrimSpace(config.Endpoint)
|
||
config.Region = strings.TrimSpace(config.Region)
|
||
config.Bucket = strings.TrimSpace(config.Bucket)
|
||
if config.Region == "" {
|
||
config.Region = "us-east-1"
|
||
}
|
||
if config.MaxObjectBytes == 0 {
|
||
config.MaxObjectBytes = 100 * 1024 * 1024
|
||
}
|
||
endpoint, err := url.Parse(config.Endpoint)
|
||
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
|
||
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
|
||
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
|
||
return nil, errors.New("invalid S3 object store configuration")
|
||
}
|
||
httpClient := &http.Client{Timeout: 2 * time.Minute}
|
||
if config.HTTPClient != nil {
|
||
clone := *config.HTTPClient
|
||
httpClient = &clone
|
||
}
|
||
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
|
||
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
|
||
client := s3.New(s3.Options{
|
||
Region: config.Region,
|
||
BaseEndpoint: aws.String(endpoint.String()),
|
||
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
|
||
UsePathStyle: forcePathStyle,
|
||
HTTPClient: httpClient,
|
||
})
|
||
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
|
||
}
|
||
|
||
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
|
||
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
|
||
return nil
|
||
}
|
||
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
|
||
return errors.New("ensure release object bucket")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
|
||
if provider != store.provider || !validObjectKey(physicalPath) {
|
||
return nil, errors.New("invalid release object source")
|
||
}
|
||
objectPath, ok := providerPathFromLocator(physicalPath)
|
||
if !ok {
|
||
return nil, errors.New("invalid release object source")
|
||
}
|
||
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
||
if err != nil {
|
||
return nil, errors.New("read release source object")
|
||
}
|
||
return object.Body, nil
|
||
}
|
||
|
||
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
|
||
if provider != store.provider || !validObjectKey(objectPath) || size < 0 || size > store.maxObjectBytes {
|
||
return errors.New("invalid release object write")
|
||
}
|
||
// 真实资源路径是 storage://<backend>/oss://<bucket>/<key>:
|
||
// 落盘只写配置的桶,所以必须先把定位符拆成裸 key,不能整串当 key。
|
||
key, ok := providerPathFromLocator(objectPath)
|
||
if !ok {
|
||
return errors.New("invalid release object write")
|
||
}
|
||
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
|
||
return fmt.Errorf("write release object: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
|
||
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
|
||
if provider != store.provider || !validObjectLocator(objectPath) {
|
||
return errors.New("invalid release object delete")
|
||
}
|
||
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}); err != nil {
|
||
return errors.New("delete release object")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
|
||
if provider != store.provider || !validObjectLocator(objectPath) {
|
||
return 0, "", errors.New("invalid release object read")
|
||
}
|
||
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
||
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
|
||
return 0, "", errors.New("inspect release object")
|
||
}
|
||
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
|
||
if err != nil {
|
||
return 0, "", errors.New("read release object")
|
||
}
|
||
defer object.Body.Close()
|
||
hash := sha256.New()
|
||
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
|
||
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
|
||
return 0, "", errors.New("verify release object")
|
||
}
|
||
return written, hex.EncodeToString(hash.Sum(nil)), nil
|
||
}
|
||
|
||
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
|
||
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
|
||
return errors.New("invalid release object copy")
|
||
}
|
||
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
|
||
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
|
||
return errors.New("copy release object")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
var _ ObjectStore = (*S3ObjectStore)(nil)
|
||
var _ ObjectReader = (*S3ObjectStore)(nil)
|