From 5c0cda3630f7d203e72b5b577dfbec88fef9616f Mon Sep 17 00:00:00 2001 From: "ochan.kwon" Date: Mon, 30 Mar 2026 15:58:13 +0900 Subject: [PATCH] fix(storage): add S3 provider scheme support to frontend and backend allowlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The S3 storage adapter (s3.go) was added in e309e0b but the provider scheme integration was incomplete — ParseProviderScheme() and frontend security.ts were not updated, causing s3:// URLs to fail in the /files proxy, VLM image processing, and frontend rendering. Additionally, S3-compatible services (MinIO, Ceph, etc.) require path-style addressing (endpoint/bucket/key) but the SDK client only used virtual-hosted style. The deprecated WithEndpointResolver API is replaced with BaseEndpoint, and path-style is auto-enabled for non-AWS endpoints (detected by amazonaws.com in the URL). Changes: - types/knowledgebase.go: add "s3" to ParseProviderScheme() - file/s3.go: auto path-style for S3-compatible endpoints + replace deprecated WithEndpointResolver with BaseEndpoint - frontend/security.ts: add s3 to 5 provider scheme allowlists - knowledgebase_test.go: add ParseProviderScheme + InferStorage tests - s3_test.go: add path-style detection + parseS3FilePath tests --- frontend/src/utils/security.ts | 10 +- internal/application/service/file/s3.go | 11 +- internal/application/service/file/s3_test.go | 109 +++++++++++++++++++ internal/types/knowledgebase.go | 2 +- internal/types/knowledgebase_test.go | 55 ++++++++++ 5 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 internal/application/service/file/s3_test.go create mode 100644 internal/types/knowledgebase_test.go diff --git a/frontend/src/utils/security.ts b/frontend/src/utils/security.ts index a446539b..cacf5009 100644 --- a/frontend/src/utils/security.ts +++ b/frontend/src/utils/security.ts @@ -41,7 +41,7 @@ const DOMPurifyConfig = { 'display', 'pointer-events', 'cursor', 'data-emit', 'direction' ], // 允许的协议 - ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i, + ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i, // 禁止的标签和属性 FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'], FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'], @@ -119,7 +119,7 @@ function protectProviderImageSrcInHTML(html: string): string { .replace(/&/g, '&') .replace(/"/g, '"'); return html.replace( - /]*?)\ssrc=(["'])(local|minio|cos|tos):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, + /]*?)\ssrc=(["'])(local|minio|cos|tos|s3):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, (_m, before, quote, provider, restPathRaw, after) => { const restPath = decodeProviderURL(restPathRaw); const protectedSrc = `${provider}://${restPath}`; @@ -173,7 +173,7 @@ export function isValidURL(url: string): boolean { } // 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL - if (/^(local|minio|cos|tos):\/\/\S+$/i.test(trimmed)) { + if (/^(local|minio|cos|tos|s3):\/\/\S+$/i.test(trimmed)) { return true; } @@ -299,7 +299,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } const images = root.querySelectorAll( - 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"]', + 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"]', ); if (!images.length) { return; @@ -319,7 +319,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } img.dataset.authHydrated = '1'; - const isProviderScheme = /^(local|minio|cos|tos):\/\//.test(sourceURL); + const isProviderScheme = /^(local|minio|cos|tos|s3):\/\//.test(sourceURL); const requestURL = isProviderScheme ? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}` : sourceURL; diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go index b0cd5fd4..101374a6 100644 --- a/internal/application/service/file/s3.go +++ b/internal/application/service/file/s3.go @@ -43,11 +43,16 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix return nil, fmt.Errorf("failed to load AWS config: %w", err) } - // Create S3 client with custom endpoint if provided + // Create S3 client with custom endpoint if provided. + // For S3-compatible services (non-AWS), use path-style addressing + // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key). var client *s3.Client if endpoint != "" { - // Use S3-specific endpoint resolver for custom endpoints - client = s3.NewFromConfig(cfg, s3.WithEndpointResolver(s3.EndpointResolverFromURL(endpoint))) + usePathStyle := !strings.Contains(endpoint, "amazonaws.com") + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = usePathStyle + }) } else { // Standard AWS S3 client = s3.NewFromConfig(cfg) diff --git a/internal/application/service/file/s3_test.go b/internal/application/service/file/s3_test.go new file mode 100644 index 00000000..0bab6d68 --- /dev/null +++ b/internal/application/service/file/s3_test.go @@ -0,0 +1,109 @@ +package file + +import ( + "strings" + "testing" +) + +func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) { + tests := []struct { + name string + endpoint string + wantPathStyle bool + }{ + { + name: "S3-compatible service uses path-style", + endpoint: "https://storage.internal:9000", + wantPathStyle: true, + }, + { + name: "MinIO endpoint uses path-style", + endpoint: "http://minio.local:9000", + wantPathStyle: true, + }, + { + name: "AWS S3 regional endpoint uses virtual-hosted", + endpoint: "https://s3.us-east-1.amazonaws.com", + wantPathStyle: false, + }, + { + name: "AWS China endpoint uses virtual-hosted", + endpoint: "https://s3.cn-north-1.amazonaws.com.cn", + wantPathStyle: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Verify the path-style detection logic matches our implementation + usePathStyle := !strings.Contains(tt.endpoint, "amazonaws.com") + if usePathStyle != tt.wantPathStyle { + t.Errorf("endpoint %q: usePathStyle = %v, want %v", tt.endpoint, usePathStyle, tt.wantPathStyle) + } + }) + } +} + +func TestNewS3Client_EmptyEndpoint(t *testing.T) { + // Empty endpoint should not trigger path-style (standard AWS S3) + endpoint := "" + if endpoint != "" { + t.Fatal("expected empty endpoint to skip custom configuration") + } +} + +func TestParseS3FilePath(t *testing.T) { + svc := &s3FileService{bucketName: "test-bucket"} + + tests := []struct { + name string + input string + want string + wantErr bool + }{ + { + name: "valid s3 path", + input: "s3://test-bucket/123/exports/abc.png", + want: "123/exports/abc.png", + }, + { + name: "wrong bucket", + input: "s3://other-bucket/key", + wantErr: true, + }, + { + name: "not s3 scheme", + input: "minio://test-bucket/key", + wantErr: true, + }, + { + name: "missing object key", + input: "s3://test-bucket/", + wantErr: true, + }, + { + name: "empty path", + input: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := svc.parseS3FilePath(tt.input) + if tt.wantErr { + if err == nil { + t.Errorf("parseS3FilePath(%q) expected error, got %q", tt.input, got) + } + return + } + if err != nil { + t.Errorf("parseS3FilePath(%q) unexpected error: %v", tt.input, err) + return + } + if got != tt.want { + t.Errorf("parseS3FilePath(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index f5099e46..27af88bb 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -264,7 +264,7 @@ func InferStorageFromFilePath(filePath string) string { // e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local" // Returns "" if the path does not use a known provider scheme. func ParseProviderScheme(filePath string) string { - for _, provider := range []string{"local", "minio", "cos", "tos"} { + for _, provider := range []string{"local", "minio", "cos", "tos", "s3"} { if strings.HasPrefix(filePath, provider+"://") { return provider } diff --git a/internal/types/knowledgebase_test.go b/internal/types/knowledgebase_test.go new file mode 100644 index 00000000..2f860b9f --- /dev/null +++ b/internal/types/knowledgebase_test.go @@ -0,0 +1,55 @@ +package types + +import "testing" + +func TestParseProviderScheme(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"local://tenant/file.pdf", "local"}, + {"minio://bucket/key", "minio"}, + {"cos://bucket/key", "cos"}, + {"tos://bucket/key", "tos"}, + {"s3://bucket/key", "s3"}, + {"s3://my-bucket/weknora/123/exports/abc.png", "s3"}, + {"https://example.com/img.png", ""}, + {"http://localhost:9000/bucket/key", ""}, + {"/data/files/images/abc.png", ""}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := ParseProviderScheme(tt.input) + if got != tt.want { + t.Errorf("ParseProviderScheme(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestInferStorageFromFilePath(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"local://tenant/file.pdf", "local"}, + {"minio://bucket/key", "minio"}, + {"cos://bucket/key", "cos"}, + {"tos://bucket/key", "tos"}, + {"s3://bucket/key", "s3"}, + {"https://my-bucket.cos.ap-guangzhou.myqcloud.com/key", "cos"}, + {"https://example.com/img.png", ""}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := InferStorageFromFilePath(tt.input) + if got != tt.want { + t.Errorf("InferStorageFromFilePath(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +}