Files
knowledge-governance/internal/releaseadapter/v071/s3_object_store.go
T
姚章浩 ac3b395102
build / build (push) Successful in 1m21s
fix(library): 对齐真实 WeKnora Schema + 入口级真实服务验收
为什么:候选环境改用真实 WeKnora 表结构后,暴露出多处"按猜的列名/形状"的假设,
以及若干 varchar 长度、对象定位符、空索引基线问题,导致真实链路跑不通。

- knowledge_tag_relations 无 tenant_id/KB 列 → 归属走父表;resources 作用域=绑定+chunk 引用。
- 写入顺序按真实外键;无代理键表用内容摘要核对。
- 队列作业 ID / 版本记录 ID 改固定长度摘要(列宽 varchar(36)/varchar(64))。
- 对象定位符拆 storage://<backend>/oss://<bucket>/<key>;provider 可配置。
- 空索引基线全链路一致(Current/Export/Backup/Archive),非空仍严格比对映射。
- finalize 阶段:版本登记失败不报成功;崩溃接管复用已持久化预备份。
- 迁移 000069:同一目标的 sync/restore 跨 kind 互斥。
2026-10-28 17:53:43 +08:00

160 lines
6.3 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package v071
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3ObjectStoreConfig struct {
Provider string
Endpoint string
Region string
AccessKeyID string
SecretAccessKey string
Bucket string
Secure bool
ForcePathStyle bool
MaxObjectBytes int64
HTTPClient *http.Client
}
type S3ObjectStore struct {
provider string
bucket string
maxObjectBytes int64
client *s3.Client
}
func NewS3ObjectStore(config S3ObjectStoreConfig) (*S3ObjectStore, error) {
config.Provider = strings.TrimSpace(config.Provider)
config.Endpoint = strings.TrimSpace(config.Endpoint)
config.Region = strings.TrimSpace(config.Region)
config.Bucket = strings.TrimSpace(config.Bucket)
if config.Region == "" {
config.Region = "us-east-1"
}
if config.MaxObjectBytes == 0 {
config.MaxObjectBytes = 100 * 1024 * 1024
}
endpoint, err := url.Parse(config.Endpoint)
if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Path != "" || endpoint.RawQuery != "" || endpoint.Fragment != "" ||
(endpoint.Scheme != "http" && endpoint.Scheme != "https") || (config.Secure && endpoint.Scheme != "https") ||
!validObjectProvider(config.Provider) || config.AccessKeyID == "" || config.SecretAccessKey == "" || config.Bucket == "" || config.MaxObjectBytes < 1 {
return nil, errors.New("invalid S3 object store configuration")
}
httpClient := &http.Client{Timeout: 2 * time.Minute}
if config.HTTPClient != nil {
clone := *config.HTTPClient
httpClient = &clone
}
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("S3 redirect rejected") }
forcePathStyle := config.ForcePathStyle || config.Provider == "minio"
client := s3.New(s3.Options{
Region: config.Region,
BaseEndpoint: aws.String(endpoint.String()),
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")),
UsePathStyle: forcePathStyle,
HTTPClient: httpClient,
})
return &S3ObjectStore{provider: config.Provider, bucket: config.Bucket, maxObjectBytes: config.MaxObjectBytes, client: client}, nil
}
func (store *S3ObjectStore) EnsureBucket(ctx context.Context) error {
if _, err := store.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(store.bucket)}); err == nil {
return nil
}
if _, err := store.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(store.bucket)}); err != nil {
return errors.New("ensure release object bucket")
}
return nil
}
func (store *S3ObjectStore) Open(ctx context.Context, provider, physicalPath string) (io.ReadCloser, error) {
if provider != store.provider || !validObjectKey(physicalPath) {
return nil, errors.New("invalid release object source")
}
objectPath, ok := providerPathFromLocator(physicalPath)
if !ok {
return nil, errors.New("invalid release object source")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return nil, errors.New("read release source object")
}
return object.Body, nil
}
func (store *S3ObjectStore) Put(ctx context.Context, provider, objectPath string, reader io.Reader, size int64) error {
if provider != store.provider || !validObjectKey(objectPath) || size < 0 || size > store.maxObjectBytes {
return errors.New("invalid release object write")
}
// 真实资源路径是 storage://<backend>/oss://<bucket>/<key>:
// 落盘只写配置的桶,所以必须先把定位符拆成裸 key,不能整串当 key。
key, ok := providerPathFromLocator(objectPath)
if !ok {
return errors.New("invalid release object write")
}
if _, err := store.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(key), Body: reader, ContentLength: aws.Int64(size)}); err != nil {
return fmt.Errorf("write release object: %w", err)
}
return nil
}
// Delete 删除单个对象;只用于显式清理(保留策略未接线,不会自动调用)。
func (store *S3ObjectStore) Delete(ctx context.Context, provider, objectPath string) error {
if provider != store.provider || !validObjectLocator(objectPath) {
return errors.New("invalid release object delete")
}
if _, err := store.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)}); err != nil {
return errors.New("delete release object")
}
return nil
}
func (store *S3ObjectStore) Stat(ctx context.Context, provider, objectPath string) (int64, string, error) {
if provider != store.provider || !validObjectLocator(objectPath) {
return 0, "", errors.New("invalid release object read")
}
head, err := store.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil || head.ContentLength == nil || *head.ContentLength < 0 || *head.ContentLength > store.maxObjectBytes {
return 0, "", errors.New("inspect release object")
}
object, err := store.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(objectPath)})
if err != nil {
return 0, "", errors.New("read release object")
}
defer object.Body.Close()
hash := sha256.New()
written, err := io.Copy(hash, io.LimitReader(object.Body, store.maxObjectBytes+1))
if err != nil || written != *head.ContentLength || written > store.maxObjectBytes {
return 0, "", errors.New("verify release object")
}
return written, hex.EncodeToString(hash.Sum(nil)), nil
}
func (store *S3ObjectStore) Copy(ctx context.Context, provider, sourcePath, targetPath string) error {
if provider != store.provider || !validObjectLocator(sourcePath) || !validObjectLocator(targetPath) {
return errors.New("invalid release object copy")
}
copySource := url.PathEscape(store.bucket + "/" + sourcePath)
if _, err := store.client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: aws.String(store.bucket), Key: aws.String(targetPath), CopySource: aws.String(copySource)}); err != nil {
return errors.New("copy release object")
}
return nil
}
var _ ObjectStore = (*S3ObjectStore)(nil)
var _ ObjectReader = (*S3ObjectStore)(nil)