fix(storage): add S3 provider scheme support to frontend and backend allowlists
The S3 storage adapter (s3.go) was added in e309e0b but the provider
scheme integration was incomplete — ParseProviderScheme() and frontend
security.ts were not updated, causing s3:// URLs to fail in the /files
proxy, VLM image processing, and frontend rendering.
Additionally, S3-compatible services (MinIO, Ceph, etc.) require
path-style addressing (endpoint/bucket/key) but the SDK client only
used virtual-hosted style. The deprecated WithEndpointResolver API
is replaced with BaseEndpoint, and path-style is auto-enabled for
non-AWS endpoints (detected by amazonaws.com in the URL).
Changes:
- types/knowledgebase.go: add "s3" to ParseProviderScheme()
- file/s3.go: auto path-style for S3-compatible endpoints + replace
deprecated WithEndpointResolver with BaseEndpoint
- frontend/security.ts: add s3 to 5 provider scheme allowlists
- knowledgebase_test.go: add ParseProviderScheme + InferStorage tests
- s3_test.go: add path-style detection + parseS3FilePath tests
This commit is contained in:
@@ -41,7 +41,7 @@ const DOMPurifyConfig = {
|
||||
'display', 'pointer-events', 'cursor', 'data-emit', 'direction'
|
||||
],
|
||||
// 允许的协议
|
||||
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
|
||||
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
|
||||
// 禁止的标签和属性
|
||||
FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'],
|
||||
FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'],
|
||||
@@ -119,7 +119,7 @@ function protectProviderImageSrcInHTML(html: string): string {
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"');
|
||||
return html.replace(
|
||||
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi,
|
||||
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi,
|
||||
(_m, before, quote, provider, restPathRaw, after) => {
|
||||
const restPath = decodeProviderURL(restPathRaw);
|
||||
const protectedSrc = `${provider}://${restPath}`;
|
||||
@@ -173,7 +173,7 @@ export function isValidURL(url: string): boolean {
|
||||
}
|
||||
|
||||
// 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL
|
||||
if (/^(local|minio|cos|tos):\/\/\S+$/i.test(trimmed)) {
|
||||
if (/^(local|minio|cos|tos|s3):\/\/\S+$/i.test(trimmed)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -299,7 +299,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
|
||||
}
|
||||
|
||||
const images = root.querySelectorAll<HTMLImageElement>(
|
||||
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"]',
|
||||
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"]',
|
||||
);
|
||||
if (!images.length) {
|
||||
return;
|
||||
@@ -319,7 +319,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
|
||||
}
|
||||
img.dataset.authHydrated = '1';
|
||||
|
||||
const isProviderScheme = /^(local|minio|cos|tos):\/\//.test(sourceURL);
|
||||
const isProviderScheme = /^(local|minio|cos|tos|s3):\/\//.test(sourceURL);
|
||||
const requestURL = isProviderScheme
|
||||
? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}`
|
||||
: sourceURL;
|
||||
|
||||
@@ -43,11 +43,16 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
|
||||
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client with custom endpoint if provided
|
||||
// Create S3 client with custom endpoint if provided.
|
||||
// For S3-compatible services (non-AWS), use path-style addressing
|
||||
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
|
||||
var client *s3.Client
|
||||
if endpoint != "" {
|
||||
// Use S3-specific endpoint resolver for custom endpoints
|
||||
client = s3.NewFromConfig(cfg, s3.WithEndpointResolver(s3.EndpointResolverFromURL(endpoint)))
|
||||
usePathStyle := !strings.Contains(endpoint, "amazonaws.com")
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = usePathStyle
|
||||
})
|
||||
} else {
|
||||
// Standard AWS S3
|
||||
client = s3.NewFromConfig(cfg)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
wantPathStyle bool
|
||||
}{
|
||||
{
|
||||
name: "S3-compatible service uses path-style",
|
||||
endpoint: "https://storage.internal:9000",
|
||||
wantPathStyle: true,
|
||||
},
|
||||
{
|
||||
name: "MinIO endpoint uses path-style",
|
||||
endpoint: "http://minio.local:9000",
|
||||
wantPathStyle: true,
|
||||
},
|
||||
{
|
||||
name: "AWS S3 regional endpoint uses virtual-hosted",
|
||||
endpoint: "https://s3.us-east-1.amazonaws.com",
|
||||
wantPathStyle: false,
|
||||
},
|
||||
{
|
||||
name: "AWS China endpoint uses virtual-hosted",
|
||||
endpoint: "https://s3.cn-north-1.amazonaws.com.cn",
|
||||
wantPathStyle: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Verify the path-style detection logic matches our implementation
|
||||
usePathStyle := !strings.Contains(tt.endpoint, "amazonaws.com")
|
||||
if usePathStyle != tt.wantPathStyle {
|
||||
t.Errorf("endpoint %q: usePathStyle = %v, want %v", tt.endpoint, usePathStyle, tt.wantPathStyle)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3Client_EmptyEndpoint(t *testing.T) {
|
||||
// Empty endpoint should not trigger path-style (standard AWS S3)
|
||||
endpoint := ""
|
||||
if endpoint != "" {
|
||||
t.Fatal("expected empty endpoint to skip custom configuration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseS3FilePath(t *testing.T) {
|
||||
svc := &s3FileService{bucketName: "test-bucket"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid s3 path",
|
||||
input: "s3://test-bucket/123/exports/abc.png",
|
||||
want: "123/exports/abc.png",
|
||||
},
|
||||
{
|
||||
name: "wrong bucket",
|
||||
input: "s3://other-bucket/key",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "not s3 scheme",
|
||||
input: "minio://test-bucket/key",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing object key",
|
||||
input: "s3://test-bucket/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := svc.parseS3FilePath(tt.input)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("parseS3FilePath(%q) expected error, got %q", tt.input, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseS3FilePath(%q) unexpected error: %v", tt.input, err)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("parseS3FilePath(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func InferStorageFromFilePath(filePath string) string {
|
||||
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
|
||||
// Returns "" if the path does not use a known provider scheme.
|
||||
func ParseProviderScheme(filePath string) string {
|
||||
for _, provider := range []string{"local", "minio", "cos", "tos"} {
|
||||
for _, provider := range []string{"local", "minio", "cos", "tos", "s3"} {
|
||||
if strings.HasPrefix(filePath, provider+"://") {
|
||||
return provider
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package types
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseProviderScheme(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"local://tenant/file.pdf", "local"},
|
||||
{"minio://bucket/key", "minio"},
|
||||
{"cos://bucket/key", "cos"},
|
||||
{"tos://bucket/key", "tos"},
|
||||
{"s3://bucket/key", "s3"},
|
||||
{"s3://my-bucket/weknora/123/exports/abc.png", "s3"},
|
||||
{"https://example.com/img.png", ""},
|
||||
{"http://localhost:9000/bucket/key", ""},
|
||||
{"/data/files/images/abc.png", ""},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := ParseProviderScheme(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseProviderScheme(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferStorageFromFilePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"local://tenant/file.pdf", "local"},
|
||||
{"minio://bucket/key", "minio"},
|
||||
{"cos://bucket/key", "cos"},
|
||||
{"tos://bucket/key", "tos"},
|
||||
{"s3://bucket/key", "s3"},
|
||||
{"https://my-bucket.cos.ap-guangzhou.myqcloud.com/key", "cos"},
|
||||
{"https://example.com/img.png", ""},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := InferStorageFromFilePath(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("InferStorageFromFilePath(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user