From 6d14cc3536325c603dfddf61ea8044ce7c1ce9c1 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 15 Jul 2026 16:31:21 +0800 Subject: [PATCH] feat(resource): add stable resource registry with LLM-context alias compaction Introduce resource:// as the stable application identity for stored objects, decoupling API responses, persisted rich text, and LLM prompts from physical storage paths. Adds a resource catalog service (register/resolve/bind, revocable expiring access grants), a request-local alias registry that compacts resource references while in the LLM context and restores them (including across streaming chunks), plus router, IM, chat-pipeline, and multimodal integration. Includes schema migrations (resources, resource_bindings, resource_access_grants) for sqlite, and hardens URL/output security handling. --- frontend/src/utils/markdownDomPurify.ts | 2 +- frontend/src/utils/security.test.ts | 12 + frontend/src/utils/security.ts | 4 +- internal/agent/engine.go | 3 + internal/agent/engine_test.go | 33 ++- internal/agent/think.go | 16 ++ internal/application/repository/resource.go | 97 ++++++++ .../service/chat_pipeline/chat_completion.go | 4 + .../chat_pipeline/chat_completion_stream.go | 23 ++ internal/application/service/file/dummy.go | 5 +- .../service/file/resource_catalog.go | 193 +++++++++++++++ .../service/file/resource_catalog_test.go | 107 ++++++++ .../application/service/image_multimodal.go | 21 +- internal/application/service/resource.go | 209 ++++++++++++++++ internal/application/service/resource_test.go | 73 ++++++ .../application/service/storagebackend.go | 71 +++++- internal/container/container.go | 44 +++- internal/im/service.go | 11 +- internal/llmresource/registry.go | 176 +++++++++++++ internal/llmresource/registry_test.go | 43 ++++ internal/models/chat/image_resolve.go | 16 +- internal/router/router.go | 231 ++++++++++++++++-- internal/router/router_files_test.go | 122 +++++++++ internal/router/router_static_test.go | 30 +++ internal/types/interfaces/resource.go | 45 +++- internal/types/knowledgebase.go | 2 +- internal/types/resource.go | 131 ++++++++++ internal/utils/security.go | 11 +- migrations/sqlite/000000_init.down.sql | 3 + migrations/sqlite/000000_init.up.sql | 53 ++++ .../000069_resource_registry.down.sql | 3 + .../versioned/000069_resource_registry.up.sql | 55 +++++ 32 files changed, 1790 insertions(+), 59 deletions(-) create mode 100644 internal/application/repository/resource.go create mode 100644 internal/application/service/file/resource_catalog.go create mode 100644 internal/application/service/file/resource_catalog_test.go create mode 100644 internal/application/service/resource.go create mode 100644 internal/application/service/resource_test.go create mode 100644 internal/llmresource/registry.go create mode 100644 internal/llmresource/registry_test.go create mode 100644 internal/router/router_static_test.go create mode 100644 internal/types/resource.go create mode 100644 migrations/versioned/000069_resource_registry.down.sql create mode 100644 migrations/versioned/000069_resource_registry.up.sql diff --git a/frontend/src/utils/markdownDomPurify.ts b/frontend/src/utils/markdownDomPurify.ts index 8b97d959..98354e3c 100644 --- a/frontend/src/utils/markdownDomPurify.ts +++ b/frontend/src/utils/markdownDomPurify.ts @@ -2,7 +2,7 @@ export const domPurifyForbidTags = ['script', 'style', 'object', 'embed', 'form' export const domPurifyForbidAttr = ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'] as const; export const domPurifyAllowedUriRegexp = - /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|(?:storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i; + /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|(?:resource|storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i; /** Shared DOMPurify security options (FORBID_*, URI scheme, DOM flags). */ export const domPurifySecurityOptions = { diff --git a/frontend/src/utils/security.test.ts b/frontend/src/utils/security.test.ts index 743c10bf..5df0dd4d 100644 --- a/frontend/src/utils/security.test.ts +++ b/frontend/src/utils/security.test.ts @@ -24,3 +24,15 @@ test('protectProviderImageSrcInHTML uses a placeholder src for storage-backend i /data-protected-src="storage:\/\/c0d93536-702c-4977-aa5e-fe670073c3cb\/local:\/\/10000\/exports\/a\.png"/, ) }) + +test('protectProviderImageSrcInHTML uses a placeholder src for resource references', () => { + const html = '

preview

' + const sanitized = protectProviderImageSrcInHTML(html) + const renderedSrc = sanitized.match(/]*\ssrc="([^"]+)"/)?.[1] + + assert.match(renderedSrc || '', /^data:image\/gif;base64,/) + assert.match( + sanitized, + /data-protected-src="resource:\/\/AbCdEfGhIjKlMnOpQrStUv"/, + ) +}) diff --git a/frontend/src/utils/security.ts b/frontend/src/utils/security.ts index ee1f8251..797fc8cc 100644 --- a/frontend/src/utils/security.ts +++ b/frontend/src/utils/security.ts @@ -12,7 +12,7 @@ import { } from './markdownDomPurify.ts'; const PROVIDER_IMAGE_PLACEHOLDER = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; -const PROVIDER_SCHEME_PATTERN = 'local|minio|cos|tos|s3|oss|ks3|obs'; +const PROVIDER_SCHEME_PATTERN = 'resource|local|minio|cos|tos|s3|oss|ks3|obs'; const PROVIDER_FILE_SCHEME_RE = new RegExp(`^(${PROVIDER_SCHEME_PATTERN}):\\/\\/\\S+$`, 'i'); const STORAGE_BACKEND_FILE_SCHEME_RE = new RegExp( `^storage:\\/\\/[0-9A-Za-z_-]+\\/(${PROVIDER_SCHEME_PATTERN}):\\/\\/\\S+$`, @@ -499,7 +499,7 @@ export async function hydrateProtectedFileImages( } const images = root.querySelectorAll( - 'img[data-protected-src], img[src^="storage://"], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"], img[src^="obs://"]', + 'img[data-protected-src], img[src^="resource://"], img[src^="storage://"], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"], img[src^="obs://"]', ); if (!images.length) { return; diff --git a/internal/agent/engine.go b/internal/agent/engine.go index a8710a47..6cbc03e3 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -14,6 +14,7 @@ import ( "github.com/Tencent/WeKnora/internal/common" appconfig "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/event" + "github.com/Tencent/WeKnora/internal/llmresource" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/tracing/langfuse" @@ -49,6 +50,7 @@ type AgentEngine struct { memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional) lastUsage types.TokenUsage // Token usage from the most recent LLM call lastSentMsgCount int // Number of messages sent in the most recent LLM call + resourceRefs *llmresource.Registry // request-local aliases for durable resource references } // ImageDescriberFunc generates a text description of an image. @@ -83,6 +85,7 @@ func NewAgentEngine( sessionID: sessionID, systemPromptTemplate: systemPromptTemplate, tokenEstimator: tokenEst, + resourceRefs: llmresource.NewRegistry(), } // Initialize memory consolidator if context window management is configured diff --git a/internal/agent/engine_test.go b/internal/agent/engine_test.go index f05e56c9..c8900262 100644 --- a/internal/agent/engine_test.go +++ b/internal/agent/engine_test.go @@ -24,16 +24,22 @@ type mockResponse struct { type mockChat struct { mu sync.Mutex responses []mockResponse + calls [][]chat.Message callCount int } -func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (<-chan types.StreamResponse, error) { +func (m *mockChat) ChatStream( + _ context.Context, + messages []chat.Message, + _ *chat.ChatOptions, +) (<-chan types.StreamResponse, error) { m.mu.Lock() defer m.mu.Unlock() if m.callCount >= len(m.responses) { return nil, fmt.Errorf("unexpected ChatStream call #%d (only %d responses prepared)", m.callCount, len(m.responses)) } resp := m.responses[m.callCount] + m.calls = append(m.calls, append([]chat.Message(nil), messages...)) m.callCount++ ch := make(chan types.StreamResponse, len(resp.chunks)) @@ -44,6 +50,25 @@ func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatO return ch, nil } +func TestStreamLLMResourceAliasesRoundTrip(t *testing.T) { + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{ + {ResponseType: types.ResponseTypeAnswer, Content: "![image](res:0"}, + {ResponseType: types.ResponseTypeAnswer, Content: "001)", Done: true}, + }}}} + engine := newTestEngine(t, model) + result, err := engine.streamLLMToEventBus( + context.Background(), + []chat.Message{{Role: "tool", Content: "source=" + ref}}, + nil, + nil, + ) + require.NoError(t, err) + require.Equal(t, "![image]("+ref+")", result.Content) + require.Len(t, model.calls, 1) + require.Equal(t, "source=res:0001", model.calls[0][0].Content) +} + func (m *mockChat) Chat(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (*types.ChatResponse, error) { return nil, fmt.Errorf("not implemented") } @@ -270,8 +295,10 @@ func TestStreamThinkingToEventBus_SplitsInlineThinkBlock(t *testing.T) { mock := &mockChat{ responses: []mockResponse{ {chunks: []types.StreamResponse{ - {ResponseType: types.ResponseTypeAnswer, Content: "hidden reasoningVisible answer.", - Done: true, FinishReason: "stop"}, + { + ResponseType: types.ResponseTypeAnswer, Content: "hidden reasoningVisible answer.", + Done: true, FinishReason: "stop", + }, }}, }, } diff --git a/internal/agent/think.go b/internal/agent/think.go index aaafd2aa..4b92de7e 100644 --- a/internal/agent/think.go +++ b/internal/agent/think.go @@ -9,6 +9,7 @@ import ( agenttools "github.com/Tencent/WeKnora/internal/agent/tools" "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/event" + "github.com/Tencent/WeKnora/internal/llmresource" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" @@ -37,6 +38,7 @@ func (e *AgentEngine) streamLLMToEventBus( llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout()) defer llmCancel() + messages = e.resourceRefs.EncodeMessages(messages) stream, err := e.chatModel.ChatStream(llmCtx, messages, opts) if err != nil { logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err) @@ -47,6 +49,8 @@ func (e *AgentEngine) streamLLMToEventBus( chunkCount := 0 responseTypeCounts := make(map[string]int) firstChunkTime := time.Time{} + answerDecoder := llmresource.NewStreamDecoder(e.resourceRefs) + thinkingDecoder := llmresource.NewStreamDecoder(e.resourceRefs) for chunk := range stream { chunkCount++ @@ -62,6 +66,12 @@ func (e *AgentEngine) streamLLMToEventBus( result.StreamError = chunk.Content continue } + if chunk.ResponseType == types.ResponseTypeThinking { + chunk.Content = thinkingDecoder.Feed(chunk.Content) + } else { + chunk.Content = answerDecoder.Feed(chunk.Content) + } + e.resourceRefs.DecodeToolCalls(chunk.ToolCalls) if chunk.Content != "" { isExtracted := chunk.Data != nil && chunk.Data["source"] != nil @@ -90,6 +100,12 @@ func (e *AgentEngine) streamLLMToEventBus( emitFunc(&chunk, result.Content) } } + result.Content += answerDecoder.Flush() + result.ReasoningContent += thinkingDecoder.Flush() + // Some providers stream tool-call argument fragments but expose the + // accumulated call in the final chunk. Decode once more after assembly so + // aliases split across provider chunks cannot leak into tool execution. + e.resourceRefs.DecodeToolCalls(result.ToolCalls) // Stream diagnostic summary: helps identify non-streaming patterns streamDuration := time.Duration(0) diff --git a/internal/application/repository/resource.go b/internal/application/repository/resource.go new file mode 100644 index 00000000..9ac0c431 --- /dev/null +++ b/internal/application/repository/resource.go @@ -0,0 +1,97 @@ +package repository + +import ( + "context" + "errors" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type resourceRepository struct{ db *gorm.DB } + +// NewResourceRepository creates the persistence adapter for resource metadata. +func NewResourceRepository(db *gorm.DB) interfaces.ResourceRepository { + return &resourceRepository{db: db} +} + +func (r *resourceRepository) Create(ctx context.Context, resource *types.StoredResource) error { + return r.db.WithContext(ctx).Create(resource).Error +} + +func (r *resourceRepository) GetByID(ctx context.Context, id string) (*types.StoredResource, error) { + var resource types.StoredResource + err := r.db.WithContext(ctx).Where("id = ? AND state = ?", id, types.ResourceStateActive).First(&resource).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &resource, err +} + +func (r *resourceRepository) GetByHandle(ctx context.Context, handle string) (*types.StoredResource, error) { + var resource types.StoredResource + err := r.db.WithContext(ctx). + Where("handle = ? AND state = ?", handle, types.ResourceStateActive). + First(&resource).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &resource, err +} + +func (r *resourceRepository) GetByTenantLocation( + ctx context.Context, + tenantID uint64, + locationHash string, +) (*types.StoredResource, error) { + var resource types.StoredResource + err := r.db.WithContext(ctx). + Where( + "tenant_id = ? AND location_hash = ? AND state = ?", + tenantID, + locationHash, + types.ResourceStateActive, + ). + First(&resource).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &resource, err +} + +func (r *resourceRepository) MarkDeleted(ctx context.Context, id string) error { + return r.db.WithContext(ctx).Model(&types.StoredResource{}).Where("id = ?", id). + Updates(map[string]interface{}{"state": types.ResourceStateDeleted, "deleted_at": time.Now()}).Error +} + +func (r *resourceRepository) CreateBinding(ctx context.Context, binding *types.ResourceBinding) error { + return r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(binding).Error +} + +func (r *resourceRepository) CreateGrant(ctx context.Context, grant *types.ResourceAccessGrant) error { + return r.db.WithContext(ctx).Create(grant).Error +} + +func (r *resourceRepository) GetValidGrant( + ctx context.Context, + tokenHash string, + now time.Time, +) (*types.ResourceAccessGrant, error) { + var grant types.ResourceAccessGrant + err := r.db.WithContext(ctx). + Where("token_hash = ? AND revoked_at IS NULL AND expires_at > ?", tokenHash, now). + First(&grant).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return &grant, err +} + +func (r *resourceRepository) DeleteExpiredGrants(ctx context.Context, before time.Time) error { + return r.db.WithContext(ctx). + Where("expires_at <= ? OR revoked_at IS NOT NULL", before). + Delete(&types.ResourceAccessGrant{}).Error +} diff --git a/internal/application/service/chat_pipeline/chat_completion.go b/internal/application/service/chat_pipeline/chat_completion.go index 660dc0f3..71cd636f 100644 --- a/internal/application/service/chat_pipeline/chat_completion.go +++ b/internal/application/service/chat_pipeline/chat_completion.go @@ -3,6 +3,7 @@ package chatpipeline import ( "context" + "github.com/Tencent/WeKnora/internal/llmresource" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -51,6 +52,8 @@ func (p *PluginChatCompletion) OnEvent( "message_count": len(chatManage.History) + 2, }) chatMessages := prepareMessagesWithHistory(chatManage) + resourceRefs := llmresource.NewRegistry() + chatMessages = resourceRefs.EncodeMessages(chatMessages) // Call the chat model to generate response pipelineInfo(ctx, "Completion", "model_call", map[string]interface{}{ @@ -64,6 +67,7 @@ func (p *PluginChatCompletion) OnEvent( }) return ErrModelCall.WithError(err) } + resourceRefs.DecodeResponse(chatResponse) pipelineInfo(ctx, "Completion", "output", map[string]interface{}{ "answer_preview": chatResponse.Content, diff --git a/internal/application/service/chat_pipeline/chat_completion_stream.go b/internal/application/service/chat_pipeline/chat_completion_stream.go index 9f519568..5424cf12 100644 --- a/internal/application/service/chat_pipeline/chat_completion_stream.go +++ b/internal/application/service/chat_pipeline/chat_completion_stream.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/Tencent/WeKnora/internal/event" + "github.com/Tencent/WeKnora/internal/llmresource" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" @@ -55,6 +56,8 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, // Prepare base messages without history chatMessages := prepareMessagesWithHistory(chatManage) + resourceRefs := llmresource.NewRegistry() + chatMessages = resourceRefs.EncodeMessages(chatMessages) pipelineInfo(ctx, "Stream", "messages_ready", map[string]interface{}{ "message_count": len(chatMessages), "system_prompt": chatMessages[0].Content, @@ -105,6 +108,8 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, // The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled // and the upstream channel is not closed promptly. go func() { + answerDecoder := llmresource.NewStreamDecoder(resourceRefs) + thinkingDecoder := llmresource.NewStreamDecoder(resourceRefs) thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8]) answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8]) thinkingOpen := false @@ -135,6 +140,22 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, case response, ok := <-responseChan: if !ok { + if tail := thinkingDecoder.Flush(); tail != "" { + _ = eventBus.Emit(ctx, types.Event{ + ID: thinkingID, + Type: types.EventType(event.EventAgentThought), + SessionID: chatManage.SessionID, + Data: event.AgentThoughtData{Content: tail}, + }) + } + if tail := answerDecoder.Flush(); tail != "" { + _ = eventBus.Emit(ctx, types.Event{ + ID: answerID, + Type: types.EventType(event.EventAgentFinalAnswer), + SessionID: chatManage.SessionID, + Data: event.AgentFinalAnswerData{Content: tail}, + }) + } closeThinking() pipelineInfo(ctx, "Stream", "channel_close", map[string]interface{}{ "session_id": chatManage.SessionID, @@ -161,6 +182,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, } if response.ResponseType == types.ResponseTypeThinking { + response.Content = thinkingDecoder.Feed(response.Content) if response.Content != "" { thinkingOpen = true eventBus.Emit(ctx, types.Event{ @@ -180,6 +202,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, } if response.ResponseType == types.ResponseTypeAnswer { + response.Content = answerDecoder.Feed(response.Content) closeThinking() eventBus.Emit(ctx, types.Event{ ID: answerID, diff --git a/internal/application/service/file/dummy.go b/internal/application/service/file/dummy.go index 63df6028..2a89efe6 100644 --- a/internal/application/service/file/dummy.go +++ b/internal/application/service/file/dummy.go @@ -3,6 +3,7 @@ package file import ( "context" "errors" + "fmt" "io" "mime/multipart" @@ -30,7 +31,7 @@ func NewDummyFileService() interfaces.FileService { func (s *DummyFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string, ) (string, error) { - return uuid.New().String(), nil + return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil } // GetFile always returns an error as dummy service doesn't store files @@ -45,7 +46,7 @@ func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) erro // SaveBytes pretends to save bytes but just returns a random UUID func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { - return uuid.New().String(), nil + return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil } // CopyFile is a no-op for the dummy service: it logs a warning and returns the diff --git a/internal/application/service/file/resource_catalog.go b/internal/application/service/file/resource_catalog.go new file mode 100644 index 00000000..ca1e6df2 --- /dev/null +++ b/internal/application/service/file/resource_catalog.go @@ -0,0 +1,193 @@ +package file + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "mime" + "mime/multipart" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// resourceCatalogFileService keeps provider drivers physical-path-only while +// exposing stable resource:// references to the application layer. +type resourceCatalogFileService struct { + inner interfaces.FileService + catalog interfaces.ResourceCatalog + externalURL string +} + +// NewResourceCatalogFileService decorates a physical FileService with stable +// resource registration and resolution. +func NewResourceCatalogFileService( + inner interfaces.FileService, + catalog interfaces.ResourceCatalog, +) interfaces.FileService { + if inner == nil || catalog == nil { + return inner + } + return &resourceCatalogFileService{ + inner: inner, + catalog: catalog, + externalURL: strings.TrimRight(strings.TrimSpace(os.Getenv("APP_EXTERNAL_URL")), "/"), + } +} + +func (s *resourceCatalogFileService) CheckConnectivity(ctx context.Context) error { + return s.inner.CheckConnectivity(ctx) +} + +func resourceKind(name string) (string, string) { + mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name))) + kind := "file" + switch { + case strings.HasPrefix(mimeType, "image/"): + kind = "image" + case strings.HasPrefix(mimeType, "audio/"): + kind = "audio" + case strings.HasPrefix(mimeType, "video/"): + kind = "video" + } + return kind, mimeType +} + +func (s *resourceCatalogFileService) register( + ctx context.Context, + physical string, + tenantID uint64, + name string, + size int64, + temporary bool, + contentHash string, +) (string, error) { + kind, mimeType := resourceKind(name) + ref, err := s.catalog.Register(ctx, tenantID, physical, interfaces.ResourceRegistration{ + Kind: kind, + MimeType: mimeType, + OriginalName: filepath.Base(name), + Size: size, + ContentHash: contentHash, + Temporary: temporary, + }) + if err != nil { + _ = s.inner.DeleteFile(ctx, physical) + return "", fmt.Errorf("register stored resource: %w", err) + } + return ref, nil +} + +func (s *resourceCatalogFileService) SaveFile( + ctx context.Context, + file *multipart.FileHeader, + tenantID uint64, + knowledgeID string, +) (string, error) { + physical, err := s.inner.SaveFile(ctx, file, tenantID, knowledgeID) + if err != nil { + return "", err + } + ref, err := s.register(ctx, physical, tenantID, file.Filename, file.Size, false, "") + if err != nil { + return "", err + } + if knowledgeID != "" { + if err := s.catalog.Bind(ctx, ref, "knowledge", knowledgeID, "source_file"); err != nil { + _ = s.DeleteFile(ctx, ref) + return "", fmt.Errorf("bind stored resource: %w", err) + } + } + return ref, nil +} + +func (s *resourceCatalogFileService) SaveBytes( + ctx context.Context, + data []byte, + tenantID uint64, + fileName string, + temp bool, +) (string, error) { + physical, err := s.inner.SaveBytes(ctx, data, tenantID, fileName, temp) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return s.register(ctx, physical, tenantID, fileName, int64(len(data)), temp, hex.EncodeToString(sum[:])) +} + +func (s *resourceCatalogFileService) resolve(ctx context.Context, value string) (string, bool, error) { + physical, resource, err := s.catalog.ResolvePath(ctx, value) + return physical, resource != nil, err +} + +func (s *resourceCatalogFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + physical, _, err := s.resolve(ctx, filePath) + if err != nil { + return nil, err + } + return s.inner.GetFile(ctx, physical) +} + +func (s *resourceCatalogFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + physical, isResource, err := s.resolve(ctx, filePath) + if err != nil { + return "", err + } + if isResource && s.externalURL != "" { + token, grantErr := s.catalog.CreateAccessGrant(ctx, filePath, 2*time.Hour) + if grantErr != nil { + return "", grantErr + } + return s.externalURL + "/r/" + token, nil + } + return s.inner.GetFileURL(ctx, physical) +} + +func (s *resourceCatalogFileService) DeleteFile(ctx context.Context, filePath string) error { + physical, isResource, err := s.resolve(ctx, filePath) + if err != nil { + return err + } + if err := s.inner.DeleteFile(ctx, physical); err != nil { + return err + } + if isResource { + return s.catalog.MarkDeleted(ctx, filePath) + } + return nil +} + +func (s *resourceCatalogFileService) CopyFile( + ctx context.Context, + filePath string, + tenantID uint64, + knowledgeID string, +) (string, error) { + physical, _, err := s.resolve(ctx, filePath) + if err != nil { + return "", err + } + copied, err := s.inner.CopyFile(ctx, physical, tenantID, knowledgeID) + if err != nil { + return "", err + } + ref, err := s.register(ctx, copied, tenantID, filepath.Base(physical), 0, false, "") + if err != nil { + return "", err + } + if knowledgeID != "" { + if err := s.catalog.Bind(ctx, ref, "knowledge", knowledgeID, "source_file"); err != nil { + _ = s.DeleteFile(ctx, ref) + return "", fmt.Errorf("bind copied resource: %w", err) + } + } + return ref, nil +} + +var _ interfaces.FileService = (*resourceCatalogFileService)(nil) diff --git a/internal/application/service/file/resource_catalog_test.go b/internal/application/service/file/resource_catalog_test.go new file mode 100644 index 00000000..80b1292f --- /dev/null +++ b/internal/application/service/file/resource_catalog_test.go @@ -0,0 +1,107 @@ +package file + +import ( + "context" + "io" + "mime/multipart" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +type catalogStub struct { + resource *types.StoredResource + ref string +} + +func (c *catalogStub) Register( + _ context.Context, + tenantID uint64, + physicalPath string, + meta interfaces.ResourceRegistration, +) (string, error) { + c.resource = &types.StoredResource{ + ID: "resource-1", + Handle: "AbCdEfGhIjKlMnOpQrStUv", + TenantID: tenantID, + PhysicalPath: physicalPath, + OriginalName: meta.OriginalName, + } + c.ref = types.BuildResourcePath(c.resource.Handle) + return c.ref, nil +} + +func (c *catalogStub) Resolve(_ context.Context, _ string) (*types.StoredResource, error) { + return c.resource, nil +} + +func (c *catalogStub) ResolvePath(_ context.Context, value string) (string, *types.StoredResource, error) { + if value == c.ref { + return c.resource.PhysicalPath, c.resource, nil + } + return value, nil, nil +} +func (c *catalogStub) Bind(context.Context, string, string, string, string) error { return nil } +func (c *catalogStub) MarkDeleted(context.Context, string) error { return nil } +func (c *catalogStub) CreateAccessGrant(context.Context, string, time.Duration) (string, error) { + return "GrantTokenAbCdEfGhIjKl", nil +} + +func (c *catalogStub) ResolveAccessGrant(context.Context, string) (*types.StoredResource, error) { + return c.resource, nil +} + +type physicalFileStub struct { + savedPath string + readPath string +} + +func (s *physicalFileStub) CheckConnectivity(context.Context) error { return nil } +func (s *physicalFileStub) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) { + return "", nil +} + +func (s *physicalFileStub) SaveBytes(context.Context, []byte, uint64, string, bool) (string, error) { + return s.savedPath, nil +} + +func (s *physicalFileStub) GetFile(_ context.Context, path string) (io.ReadCloser, error) { + s.readPath = path + return io.NopCloser(strings.NewReader("body")), nil +} +func (s *physicalFileStub) GetFileURL(context.Context, string) (string, error) { return "", nil } +func (s *physicalFileStub) DeleteFile(context.Context, string) error { return nil } +func (s *physicalFileStub) CopyFile(context.Context, string, uint64, string) (string, error) { + return "", nil +} + +func TestResourceCatalogFileServiceReturnsReferenceAndResolvesReads(t *testing.T) { + inner := &physicalFileStub{savedPath: "local://7/exports/a.png"} + catalog := &catalogStub{} + svc := NewResourceCatalogFileService(inner, catalog) + + ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false) + require.NoError(t, err) + require.Equal(t, "resource://AbCdEfGhIjKlMnOpQrStUv", ref) + reader, err := svc.GetFile(context.Background(), ref) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Equal(t, inner.savedPath, inner.readPath) +} + +func TestResourceCatalogFileServiceReturnsShortExternalGrantURL(t *testing.T) { + t.Setenv("APP_EXTERNAL_URL", "https://weknora.example.com/") + inner := &physicalFileStub{savedPath: "local://7/exports/a.png"} + catalog := &catalogStub{} + svc := NewResourceCatalogFileService(inner, catalog) + + ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false) + require.NoError(t, err) + externalURL, err := svc.GetFileURL(context.Background(), ref) + require.NoError(t, err) + require.Equal(t, "https://weknora.example.com/r/GrantTokenAbCdEfGhIjKl", externalURL) +} diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index ca003cd5..64e471f6 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -80,6 +80,10 @@ type ImageMultimodalService struct { // fallback in knowledgeService.resolveFileService. fileSvc interfaces.FileService storageResolver interfaces.StorageBackendResolver + // resourceCatalog resolves resource:// references to their owning storage + // backend so multimodal reads target the resource's real backend instead of + // the knowledge base's currently configured one. + resourceCatalog interfaces.ResourceCatalog // spanTracker records this image's subspan under the parent attempt's // multimodal stage. nil-safe — falls back to no-op via tracker(). @@ -99,6 +103,7 @@ func NewImageMultimodalService( redisClient *redis.Client, fileSvc interfaces.FileService, storageResolver interfaces.StorageBackendResolver, + resourceCatalog interfaces.ResourceCatalog, spanTracker SpanTracker, ) interfaces.TaskHandler { return &ImageMultimodalService{ @@ -114,6 +119,7 @@ func NewImageMultimodalService( redisClient: redisClient, fileSvc: fileSvc, storageResolver: storageResolver, + resourceCatalog: resourceCatalog, spanTracker: spanTracker, } } @@ -554,6 +560,18 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex backendID, _, _ := types.ParseStorageBackendPath(payload.ImageURL) provider := types.ParseProviderScheme(payload.ImageURL) + // A resource:// reference carries no provider/backend in the URL itself; the + // authoritative backend lives on the stored resource record. Using the KB's + // currently configured backend here would break reads when the resource was + // stored on a different backend (multi-backend / post-migration). + if _, isResourceRef := types.ParseResourcePath(payload.ImageURL); isResourceRef && s.resourceCatalog != nil { + if resource, resErr := s.resourceCatalog.Resolve(ctx, payload.ImageURL); resErr != nil { + logger.Warnf(ctx, "[ImageMultimodal] resolve resource reference failed: url=%s err=%v", payload.ImageURL, resErr) + } else if resource != nil { + backendID = resource.StorageBackendID + provider = strings.ToLower(strings.TrimSpace(resource.Provider)) + } + } if provider == "" { kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if kbErr != nil { @@ -590,7 +608,8 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex // file before falling back to the URL. // - For plain http(s):// URLs it uses the SSRF-safe downloader. func (s *ImageMultimodalService) readImageBytes(ctx context.Context, payload types.ImageMultimodalPayload) ([]byte, error) { - if types.ParseProviderScheme(payload.ImageURL) != "" { + _, isResourceRef := types.ParseResourcePath(payload.ImageURL) + if isResourceRef || types.ParseProviderScheme(payload.ImageURL) != "" { fileSvc := s.resolveFileServiceForPayload(ctx, payload) if fileSvc == nil { return nil, fmt.Errorf("no file service available for %s", payload.ImageURL) diff --git a/internal/application/service/resource.go b/internal/application/service/resource.go new file mode 100644 index 00000000..a837d70a --- /dev/null +++ b/internal/application/service/resource.go @@ -0,0 +1,209 @@ +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +const defaultResourceGrantTTL = 2 * time.Hour + +type resourceCatalog struct { + repo interfaces.ResourceRepository +} + +// NewResourceCatalog creates the stable resource-reference domain service. +func NewResourceCatalog(repo interfaces.ResourceRepository) interfaces.ResourceCatalog { + return &resourceCatalog{repo: repo} +} + +func randomResourceToken() (string, error) { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} + +func resourceLocationHash(path string) string { + sum := sha256.Sum256([]byte(path)) + return hex.EncodeToString(sum[:]) +} + +func (s *resourceCatalog) Register( + ctx context.Context, + tenantID uint64, + physicalPath string, + meta interfaces.ResourceRegistration, +) (string, error) { + physicalPath = strings.TrimSpace(physicalPath) + if tenantID == 0 || physicalPath == "" { + return "", fmt.Errorf("resource registration requires tenant and physical path") + } + if _, ok := types.ParseResourcePath(physicalPath); ok { + return physicalPath, nil + } + locationHash := resourceLocationHash(physicalPath) + existing, err := s.repo.GetByTenantLocation(ctx, tenantID, locationHash) + if err != nil { + return "", err + } + if existing != nil { + return types.BuildResourcePath(existing.Handle), nil + } + + backendID, inner, scoped := types.ParseStorageBackendPath(physicalPath) + providerPath := physicalPath + if scoped { + providerPath = inner + } + provider := types.ParseProviderScheme(providerPath) + if provider == "" { + return "", fmt.Errorf("resource physical path has unsupported provider scheme") + } + lifecycle := types.ResourceLifecyclePersistent + if meta.Temporary { + lifecycle = types.ResourceLifecycleTemporary + } + for attempt := 0; attempt < 4; attempt++ { + handle, tokenErr := randomResourceToken() + if tokenErr != nil { + return "", tokenErr + } + resource := &types.StoredResource{ + Handle: handle, + TenantID: tenantID, + StorageBackendID: backendID, + Provider: provider, + PhysicalPath: physicalPath, + LocationHash: locationHash, + Kind: meta.Kind, + MimeType: meta.MimeType, + OriginalName: meta.OriginalName, + Size: meta.Size, + ContentHash: meta.ContentHash, + Lifecycle: lifecycle, + } + if err := s.repo.Create(ctx, resource); err == nil { + return types.BuildResourcePath(handle), nil + } else if !strings.Contains(strings.ToLower(err.Error()), "unique") { + return "", err + } + existing, lookupErr := s.repo.GetByTenantLocation(ctx, tenantID, locationHash) + if lookupErr == nil && existing != nil { + return types.BuildResourcePath(existing.Handle), nil + } + } + return "", fmt.Errorf("failed to allocate unique resource handle") +} + +func (s *resourceCatalog) Resolve(ctx context.Context, reference string) (*types.StoredResource, error) { + handle, ok := types.ParseResourcePath(reference) + if !ok { + return nil, fmt.Errorf("invalid resource reference") + } + resource, err := s.repo.GetByHandle(ctx, handle) + if err != nil { + return nil, err + } + if resource == nil { + return nil, fmt.Errorf("resource not found") + } + return resource, nil +} + +func (s *resourceCatalog) ResolvePath(ctx context.Context, value string) (string, *types.StoredResource, error) { + if _, ok := types.ParseResourcePath(value); !ok { + return value, nil, nil + } + resource, err := s.Resolve(ctx, value) + if err != nil { + return "", nil, err + } + return resource.PhysicalPath, resource, nil +} + +func (s *resourceCatalog) Bind(ctx context.Context, reference, ownerType, ownerID, relation string) error { + resource, err := s.Resolve(ctx, reference) + if err != nil { + return err + } + if strings.TrimSpace(ownerType) == "" || strings.TrimSpace(ownerID) == "" { + return fmt.Errorf("resource binding requires owner type and id") + } + if relation == "" { + relation = "attachment" + } + return s.repo.CreateBinding(ctx, &types.ResourceBinding{ + ResourceID: resource.ID, + TenantID: resource.TenantID, + OwnerType: ownerType, + OwnerID: ownerID, + Relation: relation, + }) +} + +func (s *resourceCatalog) MarkDeleted(ctx context.Context, reference string) error { + resource, err := s.Resolve(ctx, reference) + if err != nil { + return err + } + return s.repo.MarkDeleted(ctx, resource.ID) +} + +func (s *resourceCatalog) CreateAccessGrant(ctx context.Context, reference string, ttl time.Duration) (string, error) { + resource, err := s.Resolve(ctx, reference) + if err != nil { + return "", err + } + if ttl <= 0 { + ttl = defaultResourceGrantTTL + } + // Opportunistic cleanup keeps high-volume IM rendering from accumulating + // expired capability rows; failure is non-fatal to the current grant. + _ = s.repo.DeleteExpiredGrants(ctx, time.Now()) + for attempt := 0; attempt < 4; attempt++ { + token, tokenErr := randomResourceToken() + if tokenErr != nil { + return "", tokenErr + } + grant := &types.ResourceAccessGrant{ + TokenHash: resourceLocationHash(token), + ResourceID: resource.ID, + AccessScope: "read", + ExpiresAt: time.Now().Add(ttl), + } + if err := s.repo.CreateGrant(ctx, grant); err == nil { + return token, nil + } else if !strings.Contains(strings.ToLower(err.Error()), "unique") { + return "", err + } + } + return "", fmt.Errorf("failed to allocate unique resource access token") +} + +func (s *resourceCatalog) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) { + grant, err := s.repo.GetValidGrant(ctx, resourceLocationHash(strings.TrimSpace(token)), time.Now()) + if err != nil { + return nil, err + } + if grant == nil { + return nil, fmt.Errorf("resource access grant is invalid or expired") + } + resource, err := s.repo.GetByID(ctx, grant.ResourceID) + if err != nil { + return nil, err + } + if resource == nil { + return nil, fmt.Errorf("resource not found") + } + return resource, nil +} diff --git a/internal/application/service/resource_test.go b/internal/application/service/resource_test.go new file mode 100644 index 00000000..17e6d9a1 --- /dev/null +++ b/internal/application/service/resource_test.go @@ -0,0 +1,73 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func newResourceCatalogForTest(t *testing.T) (interfaces.ResourceCatalog, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.StoredResource{}, &types.ResourceBinding{}, &types.ResourceAccessGrant{})) + return NewResourceCatalog(repository.NewResourceRepository(db)), db +} + +func TestResourceCatalogRegisterResolveAndDeduplicate(t *testing.T) { + catalog, _ := newResourceCatalogForTest(t) + ctx := context.Background() + physical := "storage://backend-a/local://7/exports/a.png" + + ref, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{Kind: "image", OriginalName: "a.png"}) + require.NoError(t, err) + require.Regexp(t, `^resource://[0-9A-Za-z_-]{22}$`, ref) + + again, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{}) + require.NoError(t, err) + require.Equal(t, ref, again) + + resolvedPath, resource, err := catalog.ResolvePath(ctx, ref) + require.NoError(t, err) + require.Equal(t, physical, resolvedPath) + require.Equal(t, uint64(7), resource.TenantID) + require.Equal(t, "backend-a", resource.StorageBackendID) + require.Equal(t, "local", resource.Provider) +} + +func TestResourceCatalogBindingAndAccessGrant(t *testing.T) { + catalog, db := newResourceCatalogForTest(t) + ctx := context.Background() + ref, err := catalog.Register( + ctx, + 9, + "local://9/exports/report.pdf", + interfaces.ResourceRegistration{OriginalName: "report.pdf"}, + ) + require.NoError(t, err) + require.NoError(t, catalog.Bind(ctx, ref, "knowledge", "knowledge-1", "source_file")) + + token, err := catalog.CreateAccessGrant(ctx, ref, time.Minute) + require.NoError(t, err) + require.Len(t, token, 22) + var storedGrant types.ResourceAccessGrant + require.NoError(t, db.First(&storedGrant).Error) + require.NotEqual(t, token, storedGrant.TokenHash) + require.Len(t, storedGrant.TokenHash, 64) + resource, err := catalog.ResolveAccessGrant(ctx, token) + require.NoError(t, err) + require.Equal(t, uint64(9), resource.TenantID) +} + +func TestResourceCatalogRejectsUnsupportedPhysicalPath(t *testing.T) { + catalog, _ := newResourceCatalogForTest(t) + _, err := catalog.Register(context.Background(), 7, "https://example.com/a.png", interfaces.ResourceRegistration{}) + require.ErrorContains(t, err, "unsupported provider") +} diff --git a/internal/application/service/storagebackend.go b/internal/application/service/storagebackend.go index fdad850a..0676cf52 100644 --- a/internal/application/service/storagebackend.go +++ b/internal/application/service/storagebackend.go @@ -19,12 +19,35 @@ import ( ) type StorageBackendService struct { - repo interfaces.StorageBackendRepository - db *gorm.DB + repo interfaces.StorageBackendRepository + db *gorm.DB + resourceCatalog interfaces.ResourceCatalog } -func NewStorageBackendService(repo interfaces.StorageBackendRepository, db *gorm.DB) *StorageBackendService { - return &StorageBackendService{repo: repo, db: db} +// NewStorageBackendService creates a storage backend service. The optional +// catalog keeps focused tests compatible while production uses the explicit +// constructor below. +func NewStorageBackendService( + repo interfaces.StorageBackendRepository, + db *gorm.DB, + catalogs ...interfaces.ResourceCatalog, +) *StorageBackendService { + service := &StorageBackendService{repo: repo, db: db} + if len(catalogs) > 0 { + service.resourceCatalog = catalogs[0] + } + return service +} + +// NewStorageBackendServiceWithResources is the production DI constructor. +// The variadic constructor above remains convenient for focused tests that do +// not exercise resource registration. +func NewStorageBackendServiceWithResources( + repo interfaces.StorageBackendRepository, + db *gorm.DB, + catalog interfaces.ResourceCatalog, +) *StorageBackendService { + return NewStorageBackendService(repo, db, catalog) } func (s *StorageBackendService) Create(ctx context.Context, backend *types.StorageBackend) error { @@ -76,6 +99,18 @@ func (s *StorageBackendService) Update(ctx context.Context, incoming *types.Stor return err } } + if references == 0 { + if err := s.db.WithContext(ctx).Model(&types.StoredResource{}). + Where( + "tenant_id = ? AND storage_backend_id = ? AND state = ?", + incoming.TenantID, + incoming.ID, + types.ResourceStateActive, + ). + Count(&references).Error; err != nil { + return err + } + } if references > 0 { return apperrors.NewBadRequestError("a default or bound storage backend cannot be disabled") } @@ -123,6 +158,15 @@ func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id if kbCount > 0 { return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d knowledge base(s) bound to it", kbCount)) } + var resourceCount int64 + if err := tx.Model(&types.StoredResource{}). + Where("tenant_id = ? AND storage_backend_id = ? AND state = ?", tenantID, id, types.ResourceStateActive). + Count(&resourceCount).Error; err != nil { + return err + } + if resourceCount > 0 { + return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d active resource(s)", resourceCount)) + } if backend.LegacyAlias { return apperrors.NewBadRequestError("legacy storage backend cannot be deleted while old file paths may reference it") } @@ -247,12 +291,21 @@ func (s *StorageBackendService) ResolveFileService(ctx context.Context, tenant * if err != nil { return nil, provider, err } - return filesvc.NewBackendScopedFileService(backend.ID, inner), provider, nil + scoped := filesvc.NewBackendScopedFileService(backend.ID, inner) + return filesvc.NewResourceCatalogFileService(scoped, s.resourceCatalog), provider, nil } if tenant == nil { return nil, "", fmt.Errorf("workspace context missing") } - return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, localBaseDir) + inner, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig( + provider, + tenant.StorageEngineConfig, + localBaseDir, + ) + if err != nil { + return nil, resolvedProvider, err + } + return filesvc.NewResourceCatalogFileService(inner, s.resourceCatalog), resolvedProvider, nil } func validateStorageBackendEndpoint(backend *types.StorageBackend) error { @@ -276,5 +329,7 @@ func validateStorageBackendEndpoint(backend *types.StorageBackend) error { return nil } -var _ interfaces.StorageBackendService = (*StorageBackendService)(nil) -var _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil) +var ( + _ interfaces.StorageBackendService = (*StorageBackendService)(nil) + _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil) +) diff --git a/internal/container/container.go b/internal/container/container.go index a23c2702..e9d9944d 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -231,6 +231,8 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(repository.NewWebSearchProviderRepository)) must(container.Provide(repository.NewVectorStoreRepository)) must(container.Provide(repository.NewStorageBackendRepository)) + must(container.Provide(repository.NewResourceRepository)) + must(container.Provide(service.NewResourceCatalog)) // TenantStoreOwnership adapter used by the retriever factory functions // to verify that a resolved VectorStore belongs to the caller's tenant. must(container.Provide(retriever.NewVectorStoreRepoOwnership)) @@ -247,7 +249,7 @@ func BuildContainer(container *dig.Container) *dig.Container { return sr, nil })) must(container.Provide(service.NewVectorStoreService)) - must(container.Provide(service.NewStorageBackendService)) + must(container.Provide(service.NewStorageBackendServiceWithResources)) must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendService { return s })) must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendResolver { return s })) @@ -412,20 +414,38 @@ func BuildContainer(container *dig.Container) *dig.Container { // disk bytes requires rebuilding the FileService from that tenant's storage // config. The owning tenant is parsed from the URL's first path segment, which // correctly handles cross-tenant shared resources (e.g. shared KB images). -func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, storageResolver interfaces.StorageBackendResolver) { +func registerChatLocalImageResolver( + tenantRepo interfaces.TenantRepository, + storageResolver interfaces.StorageBackendResolver, + resourceCatalog interfaces.ResourceCatalog, +) { chat.LocalImageResolver = func(storageURL string) ([]byte, bool) { - tenantID := secutils.ParseTenantIDFromStoragePath(storageURL) + ctx := context.Background() + physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL) + if err != nil { + return nil, false + } + tenantID := secutils.ParseTenantIDFromStoragePath(physicalPath) + if resource != nil { + tenantID = resource.TenantID + } if tenantID == 0 { return nil, false } - ctx := context.Background() tenant, err := tenantRepo.GetTenantByID(ctx, tenantID) if err != nil || tenant == nil { return nil, false } baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) - backendID, inner, _ := types.ParseStorageBackendPath(storageURL) - provider := types.ParseProviderScheme(inner) + backendID, inner, scoped := types.ParseStorageBackendPath(physicalPath) + if resource != nil && resource.StorageBackendID != "" { + backendID = resource.StorageBackendID + } + providerPath := physicalPath + if scoped { + providerPath = inner + } + provider := types.ParseProviderScheme(providerPath) if provider == "" { provider = "local" } @@ -433,7 +453,7 @@ func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, stor if err != nil { return nil, false } - rc, err := fileSvc.GetFile(ctx, storageURL) + rc, err := fileSvc.GetFile(ctx, physicalPath) if err != nil { return nil, false } @@ -860,7 +880,15 @@ func syncSequences(db *gorm.DB) { // Returns: // - Configured file service implementation // - Error if initialization fails -func initFileService(cfg *config.Config) (interfaces.FileService, error) { +func initFileService(cfg *config.Config, catalog interfaces.ResourceCatalog) (interfaces.FileService, error) { + inner, err := initRawFileService(cfg) + if err != nil { + return nil, err + } + return file.NewResourceCatalogFileService(inner, catalog), nil +} + +func initRawFileService(_ *config.Config) (interfaces.FileService, error) { storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE")) if storageType == "" { storageType = "local" diff --git a/internal/im/service.go b/internal/im/service.go index 5bbe8353..62ee338c 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -79,7 +79,11 @@ func stripImageXMLTags(s string) string { // storageSchemeRe matches both legacy provider:// URLs and canonical // storage:///provider:// URLs. -var storageSchemeRe = regexp.MustCompile(`\b(?:storage://[0-9A-Za-z_-]+/)?(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+`) +var storageSchemeRe = regexp.MustCompile( + `\b(?:resource://[0-9A-Za-z_-]+|` + + `(?:storage://[0-9A-Za-z_-]+/)?` + + `(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+)`, +) // rewriteStorageURLs replaces all provider:// URLs in content with HTTP URLs // obtained from fileService.GetFileURL. URLs that are already HTTP or cannot @@ -128,7 +132,7 @@ func rewriteStorageURLs(ctx context.Context, content string, resolver *imFileSer // incompleteURLSuffixRe matches a provider:// URL that reaches the end of the // string — it may continue in the next chunk. var incompleteURLSuffixRe = regexp.MustCompile( - `\b(?:storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`, + `\b(?:resource|storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`, ) // findIncompleteStorageURL returns the byte offset of a potentially truncated @@ -248,6 +252,9 @@ func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileSe } func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService { + if _, ok := types.ParseResourcePath(filePath); ok { + return r.defaultSvc + } backendID, _, _ := types.ParseStorageBackendPath(filePath) provider := types.ParseProviderScheme(filePath) if provider == "" { diff --git a/internal/llmresource/registry.go b/internal/llmresource/registry.go new file mode 100644 index 00000000..b8cc97ca --- /dev/null +++ b/internal/llmresource/registry.go @@ -0,0 +1,176 @@ +// Package llmresource shortens stored-resource references while they are in an +// LLM context and restores them before application code consumes the output. +package llmresource + +import ( + "fmt" + "regexp" + "sort" + "strings" + "sync" + + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" +) + +// storedRefRE also recognizes legacy physical references. New writes persist +// resource:// handles, but old chunks and message history can still contain a +// provider URL. Giving both forms the same request-local alias makes rollout +// safe without a blocking full-table content rewrite. +var storedRefRE = regexp.MustCompile( + `resource://[0-9A-Za-z_-]{22}|` + + `(?:storage://[0-9A-Za-z_-]+/)?` + + `(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+`, +) + +// Registry assigns low-entropy, request-local aliases to stable resource +// handles. It is safe to reuse across all rounds of one Agent execution. +type Registry struct { + mu sync.RWMutex + refToAlias map[string]string + aliasToRef map[string]string +} + +// NewRegistry creates an empty request-local alias registry. +func NewRegistry() *Registry { + return &Registry{ + refToAlias: make(map[string]string), + aliasToRef: make(map[string]string), + } +} + +// EncodeText replaces stored references with compact, stable aliases. +func (r *Registry) EncodeText(value string) string { + if r == nil || value == "" { + return value + } + return storedRefRE.ReplaceAllStringFunc(value, func(ref string) string { + r.mu.Lock() + defer r.mu.Unlock() + if alias, ok := r.refToAlias[ref]; ok { + return alias + } + alias := fmt.Sprintf("res:%04d", len(r.aliasToRef)+1) + r.refToAlias[ref] = alias + r.aliasToRef[alias] = ref + return alias + }) +} + +// DecodeText restores every alias currently known to the registry. +func (r *Registry) DecodeText(value string) string { + if r == nil || value == "" { + return value + } + r.mu.RLock() + aliases := make([]string, 0, len(r.aliasToRef)) + for alias := range r.aliasToRef { + aliases = append(aliases, alias) + } + sort.SliceStable(aliases, func(i, j int) bool { return len(aliases[i]) > len(aliases[j]) }) + decoded := value + for _, alias := range aliases { + decoded = strings.ReplaceAll(decoded, alias, r.aliasToRef[alias]) + } + r.mu.RUnlock() + return decoded +} + +// EncodeMessages returns a copied message slice with textual references +// compacted. Binary/image content fields are intentionally left untouched. +func (r *Registry) EncodeMessages(messages []chat.Message) []chat.Message { + if r == nil || len(messages) == 0 { + return messages + } + encoded := make([]chat.Message, len(messages)) + copy(encoded, messages) + for i := range encoded { + encoded[i].Content = r.EncodeText(encoded[i].Content) + encoded[i].ReasoningContent = r.EncodeText(encoded[i].ReasoningContent) + if len(encoded[i].MultiContent) > 0 { + encoded[i].MultiContent = append([]chat.MessageContentPart(nil), encoded[i].MultiContent...) + for j := range encoded[i].MultiContent { + encoded[i].MultiContent[j].Text = r.EncodeText(encoded[i].MultiContent[j].Text) + } + } + if len(encoded[i].ToolCalls) > 0 { + encoded[i].ToolCalls = append([]chat.ToolCall(nil), encoded[i].ToolCalls...) + for j := range encoded[i].ToolCalls { + encoded[i].ToolCalls[j].Function.Arguments = r.EncodeText(encoded[i].ToolCalls[j].Function.Arguments) + } + } + } + return encoded +} + +// DecodeResponse restores references in a non-streaming model response. +func (r *Registry) DecodeResponse(response *types.ChatResponse) { + if r == nil || response == nil { + return + } + response.Content = r.DecodeText(response.Content) + response.ReasoningContent = r.DecodeText(response.ReasoningContent) + r.DecodeToolCalls(response.ToolCalls) +} + +// DecodeToolCalls restores aliases in tool-call JSON arguments. +func (r *Registry) DecodeToolCalls(toolCalls []types.LLMToolCall) { + for i := range toolCalls { + toolCalls[i].Function.Arguments = r.DecodeText(toolCalls[i].Function.Arguments) + } +} + +func (r *Registry) aliases() []string { + r.mu.RLock() + defer r.mu.RUnlock() + aliases := make([]string, 0, len(r.aliasToRef)) + for alias := range r.aliasToRef { + aliases = append(aliases, alias) + } + return aliases +} + +// StreamDecoder holds only a suffix that could be the beginning of a known +// alias, allowing aliases split across provider chunks to round-trip exactly. +type StreamDecoder struct { + registry *Registry + pending string +} + +// NewStreamDecoder creates an alias decoder for one streaming text channel. +func NewStreamDecoder(registry *Registry) *StreamDecoder { + return &StreamDecoder{registry: registry} +} + +// Feed decodes a chunk while retaining suffixes that may complete an alias in +// the next provider chunk. +func (d *StreamDecoder) Feed(chunk string) string { + if d == nil || d.registry == nil { + return chunk + } + combined := d.pending + chunk + d.pending = "" + hold := 0 + for _, alias := range d.registry.aliases() { + for n := 1; n < len(alias); n++ { + if n > hold && strings.HasSuffix(combined, alias[:n]) { + hold = n + } + } + } + if hold > 0 { + d.pending = combined[len(combined)-hold:] + combined = combined[:len(combined)-hold] + } + return d.registry.DecodeText(combined) +} + +// Flush returns any buffered suffix when the provider stream closes. +func (d *StreamDecoder) Flush() string { + if d == nil { + return "" + } + pending := d.pending + d.pending = "" + return d.registry.DecodeText(pending) +} diff --git a/internal/llmresource/registry_test.go b/internal/llmresource/registry_test.go new file mode 100644 index 00000000..31c45083 --- /dev/null +++ b/internal/llmresource/registry_test.go @@ -0,0 +1,43 @@ +package llmresource + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/stretchr/testify/require" +) + +func TestRegistryRoundTripAndDeduplicate(t *testing.T) { + r := NewRegistry() + ref := "resource://AbCdEfGhIjKlMnOpQrStUv" + encoded := r.EncodeText("![a](" + ref + ") and " + ref) + require.Equal(t, "![a](res:0001) and res:0001", encoded) + require.Equal(t, "![a]("+ref+") and "+ref, r.DecodeText(encoded)) +} + +func TestRegistryAliasesLegacyPhysicalReferencesDuringRollout(t *testing.T) { + r := NewRegistry() + ref := "storage://c0d93536-702c-4977-aa5e-fe670073c3cb/local://10000/exports/image.png" + encoded := r.EncodeText("![image](" + ref + ")") + require.Equal(t, "![image](res:0001)", encoded) + require.Equal(t, "![image]("+ref+")", r.DecodeText(encoded)) +} + +func TestRegistryEncodesMessageCopies(t *testing.T) { + r := NewRegistry() + ref := "resource://AbCdEfGhIjKlMnOpQrStUv" + original := []chat.Message{{Role: "tool", Content: ref}} + encoded := r.EncodeMessages(original) + require.Equal(t, ref, original[0].Content) + require.Equal(t, "res:0001", encoded[0].Content) +} + +func TestStreamDecoderRestoresSplitAlias(t *testing.T) { + r := NewRegistry() + ref := "resource://AbCdEfGhIjKlMnOpQrStUv" + require.Equal(t, "res:0001", r.EncodeText(ref)) + d := NewStreamDecoder(r) + require.Equal(t, "before ", d.Feed("before res:0")) + require.Equal(t, ref+" afte", d.Feed("001 after")) + require.Equal(t, "r", d.Flush()) +} diff --git a/internal/models/chat/image_resolve.go b/internal/models/chat/image_resolve.go index 82b263df..d0a46db6 100644 --- a/internal/models/chat/image_resolve.go +++ b/internal/models/chat/image_resolve.go @@ -12,12 +12,13 @@ import ( // resolveImageURLForLLM converts stored image paths to a format that LLM APIs can consume. // - data: URIs and http(s):// URLs are returned as-is. -// - local:// paths are read from disk and converted to base64 data URIs. +// - resource:// and provider-backed local paths are read through the +// application resolver and converted to base64 data URIs. func resolveImageURLForLLM(imageURL string) string { if strings.HasPrefix(imageURL, "data:") || strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") { return imageURL } - if strings.HasPrefix(imageURL, "local://") { + if isApplicationStoredImage(imageURL) { data := readLocalStorageBytes(imageURL) if data != nil { mime := http.DetectContentType(data) @@ -40,14 +41,21 @@ func resolveImageURLForOllama(imageURL string) []byte { } return decoded } - if strings.HasPrefix(imageURL, "local://") { + if isApplicationStoredImage(imageURL) { return readLocalStorageBytes(imageURL) } return nil } +func isApplicationStoredImage(imageURL string) bool { + return strings.HasPrefix(imageURL, "resource://") || + strings.HasPrefix(imageURL, "local://") || + strings.HasPrefix(imageURL, "storage://") +} + // LocalImageResolver, when set by the application layer at startup, resolves a -// local:// storage URL to its bytes using the owning tenant's storage config. +// resource:// or provider storage URL to bytes using the owning tenant's +// storage config. // Stored local:// URLs are relative to the storage base dir and do NOT encode // the tenant's configured PathPrefix, so a plain env-based join would miss the // prefix. When nil (e.g. in tests), callers fall back to the env-based diff --git a/internal/router/router.go b/internal/router/router.go index f93cea71..a41b24d0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -76,6 +76,7 @@ type RouterParams struct { VectorStoreHandler *handler.VectorStoreHandler StorageBackendHandler *handler.StorageBackendHandler StorageBackendResolver interfaces.StorageBackendResolver + ResourceCatalog interfaces.ResourceCatalog FAQHandler *handler.FAQHandler TagHandler *handler.TagHandler CustomAgentHandler *handler.CustomAgentHandler @@ -158,13 +159,26 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterIMRoutes(r, params.IMHandler) // Web embed 公开路由(使用 publish token 鉴权,不走全局 Auth) - RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService, params.StorageBackendResolver) + RegisterEmbedPublicRoutes( + r, + params.EmbedChannelHandler, + params.EmbedChannelService, + params.TenantService, + params.RedisClient, + params.FileService, + params.StorageBackendResolver, + params.ResourceCatalog, + ) + + // Short-lived capability URLs for IM and other clients that cannot attach + // WeKnora authentication headers. + serveResourceGrants(r, params.ResourceCatalog, params.TenantService, params.FileService, params.StorageBackendResolver) // 认证中间件 r.Use(middleware.Auth(params.TenantService, params.UserService, params.TenantMemberService, params.TenantAPIKeyService, params.Config)) // 文件服务:统一代理本地/MinIO/COS/TOS存储后端(需要认证) - serveFiles(r, params.FileService, params.StorageBackendResolver) + serveFilesWithResources(r, params.FileService, params.StorageBackendResolver, params.ResourceCatalog) // Presigned file access: no auth required, signature-verified. servePresignedFiles(r, params.TenantService, params.StorageBackendResolver) @@ -219,7 +233,14 @@ func NewRouter(params RouterParams) *gin.Engine { // KB-scoped image proxy: lets tenants render images embedded in // org-shared / agent-visible KB content, which the tenant-scoped // /files route cannot serve because it enforces same-tenant paths. - serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService, params.StorageBackendResolver) + serveKBScopedFiles( + v1, + rbacGuards, + params.TenantService, + params.FileService, + params.StorageBackendResolver, + params.ResourceCatalog, + ) RegisterKnowledgeTagRoutes(v1, params.TagHandler, rbacGuards) RegisterKnowledgeRoutes(v1, params.KnowledgeHandler, rbacGuards) RegisterFAQRoutes(v1, params.FAQHandler, rbacGuards) @@ -1295,6 +1316,7 @@ func RegisterEmbedPublicRoutes( redisClient *redis.Client, fileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver, + resourceCatalogs ...interfaces.ResourceCatalog, ) { if embedHandler == nil || embedService == nil { return @@ -1322,7 +1344,7 @@ func RegisterEmbedPublicRoutes( // Serve images embedded in bot replies (e.g. chart exports). EmbedAuth // injects the channel's tenant, and the handler enforces that the // requested path belongs to that tenant. - embed.GET("/files", newFileServeHandler(fileService, storageResolver)) + embed.GET("/files", newFileServeHandler(fileService, storageResolver, resourceCatalogs...)) } } @@ -1500,7 +1522,8 @@ func serveFrontendStatic(r *gin.Engine) { return } path := c.Request.URL.Path - if strings.HasPrefix(path, "/api/") || strings.HasPrefix(path, "/health") || strings.HasPrefix(path, "/swagger/") { + if strings.HasPrefix(path, "/api/") || strings.HasPrefix(path, "/health") || strings.HasPrefix(path, "/swagger/") || + strings.HasPrefix(path, "/r/") || path == "/files" { c.Next() return } @@ -1542,7 +1565,15 @@ type getRouteRegistrar interface { // same handler backs both the authenticated /files route and the embed route // (where EmbedAuth injects the channel's tenant). Tenant ownership of the // requested path is enforced via ValidateStoragePathTenant either way. -func newFileServeHandler(globalFileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver) gin.HandlerFunc { +func newFileServeHandler( + globalFileService interfaces.FileService, + storageResolver interfaces.StorageBackendResolver, + resourceCatalogs ...interfaces.ResourceCatalog, +) gin.HandlerFunc { + var resourceCatalog interfaces.ResourceCatalog + if len(resourceCatalogs) > 0 { + resourceCatalog = resourceCatalogs[0] + } baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR") if baseDir == "" { baseDir = "/data/files" @@ -1565,6 +1596,27 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv return } + tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant) + if tenant == nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) + return + } + resourceResolved := false + if resourceCatalog != nil { + resolvedPath, resource, err := resourceCatalog.ResolvePath(c.Request.Context(), filePath) + if err != nil { + c.Status(http.StatusNotFound) + return + } + if resource != nil { + if resource.TenantID != tenant.ID { + c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: resource not accessible"}) + return + } + filePath = resolvedPath + resourceResolved = true + } + } backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath) providerPath := filePath if scoped { @@ -1572,16 +1624,21 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv } provider := types.ParseProviderScheme(providerPath) - tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant) - if tenant == nil { - c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) - return - } - - if err := secutils.ValidateStoragePathTenant(filePath, tenant.ID); err != nil { - logger.Warnf(context.Background(), "[Router] /files denied cross-tenant or invalid path: tenant_id=%d file_path=%q err=%v", tenant.ID, filePath, err) - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"}) - return + // A registered resource's tenant is authoritative. Physical provider + // paths remain an internal locator and are not required to encode access + // control metadata (some cloud layouts contain other numeric segments). + if !resourceResolved { + if err := secutils.ValidateStoragePathTenant(filePath, tenant.ID); err != nil { + logger.Warnf( + context.Background(), + "[Router] /files denied cross-tenant or invalid path: tenant_id=%d file_path=%q err=%v", + tenant.ID, + filePath, + err, + ) + c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"}) + return + } } var ( @@ -1636,15 +1693,115 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv } func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService, resolvers ...interfaces.StorageBackendResolver) { - logger.Infof(context.Background(), "[Router] Serving files from /files") var storageResolver interfaces.StorageBackendResolver if len(resolvers) > 0 { storageResolver = resolvers[0] } + serveFilesWithResources(r, globalFileService, storageResolver, nil) +} + +func serveFilesWithResources( + r getRouteRegistrar, + globalFileService interfaces.FileService, + storageResolver interfaces.StorageBackendResolver, + resourceCatalog interfaces.ResourceCatalog, +) { + logger.Infof(context.Background(), "[Router] Serving files from /files") // /files sits outside the /api/v1 APIKeyGate; storage paths cannot be // attributed to a key's KB allow-list, so API keys are denied outright // (embed routes use their own /embed/.../files handler). - r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService, storageResolver)) + r.GET( + "/files", + middleware.DenyAPIKeyPrincipal(), + newFileServeHandler(globalFileService, storageResolver, resourceCatalog), + ) +} + +// serveResourceGrants exposes short, revocable capability URLs for clients +// such as IM platforms that cannot attach a WeKnora bearer/embed token. +func serveResourceGrants( + r *gin.Engine, + resourceCatalog interfaces.ResourceCatalog, + tenantService interfaces.TenantService, + globalFileService interfaces.FileService, + storageResolver interfaces.StorageBackendResolver, +) { + if resourceCatalog == nil || tenantService == nil { + return + } + handler := func(c *gin.Context) { + ctx := c.Request.Context() + resource, err := resourceCatalog.ResolveAccessGrant(ctx, c.Param("token")) + if err != nil || resource == nil { + c.Status(http.StatusNotFound) + return + } + tenant, err := tenantService.GetTenantByID(ctx, resource.TenantID) + if err != nil || tenant == nil { + c.Status(http.StatusNotFound) + return + } + + providerPath := resource.PhysicalPath + if _, inner, ok := types.ParseStorageBackendPath(providerPath); ok { + providerPath = inner + } + provider := types.ParseProviderScheme(providerPath) + var fileSvc interfaces.FileService + if storageResolver != nil { + fileSvc, _, err = storageResolver.ResolveFileService( + ctx, + tenant, + resource.StorageBackendID, + provider, + localStorageBaseDir(), + ) + } else { + fileSvc = globalFileService + } + if err != nil || fileSvc == nil { + logger.Warnf(ctx, "[Router] resource grant storage resolution failed: resource_id=%s err=%v", resource.ID, err) + c.Status(http.StatusNotFound) + return + } + reader, err := fileSvc.GetFile(ctx, resource.PhysicalPath) + if err != nil { + c.Status(http.StatusNotFound) + return + } + defer func() { _ = reader.Close() }() + + fileName := resource.OriginalName + if fileName == "" { + fileName = resource.PhysicalPath + } + contentType, inline := secutils.SafeContentTypeByFilename(fileName) + if resource.MimeType != "" && inline { + contentType = resource.MimeType + } + c.Header("Content-Type", contentType) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Cache-Control", "private, max-age=300") + if !inline { + c.Header("Content-Disposition", "attachment") + } + c.Status(http.StatusOK) + if c.Request.Method != http.MethodHead { + if _, err := io.Copy(c.Writer, reader); err != nil { + logger.Warnf(ctx, "[Router] resource grant write failed: resource_id=%s err=%v", resource.ID, err) + } + } + } + r.GET("/r/:token", handler) + r.HEAD("/r/:token", handler) +} + +func localStorageBaseDir() string { + baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR")) + if baseDir == "" { + baseDir = "/data/files" + } + return baseDir } // serveKBScopedFiles registers the KB-scoped file proxy used to render images @@ -1665,6 +1822,7 @@ func serveKBScopedFiles( tenantService interfaces.TenantService, globalFileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver, + resourceCatalogs ...interfaces.ResourceCatalog, ) { logger.Infof(context.Background(), "[Router] Serving KB-scoped files from /knowledge-bases/:id/files") // API keys are denied outright (as on /files): a signed URL's tenant/KB @@ -1674,7 +1832,12 @@ func serveKBScopedFiles( middleware.DenyAPIKeyPrincipal(), g.Viewer(), g.KBAccessRead("id"), - newKBScopedFileServeHandler(tenantService, globalFileService, storageResolver), + newKBScopedFileServeHandlerWithResources( + tenantService, + globalFileService, + storageResolver, + firstResourceCatalog(resourceCatalogs), + ), ) } @@ -1693,6 +1856,22 @@ func newKBScopedFileServeHandler( if len(resolvers) > 0 { storageResolver = resolvers[0] } + return newKBScopedFileServeHandlerWithResources(tenantService, globalFileService, storageResolver, nil) +} + +func firstResourceCatalog(catalogs []interfaces.ResourceCatalog) interfaces.ResourceCatalog { + if len(catalogs) == 0 { + return nil + } + return catalogs[0] +} + +func newKBScopedFileServeHandlerWithResources( + tenantService interfaces.TenantService, + globalFileService interfaces.FileService, + storageResolver interfaces.StorageBackendResolver, + resourceCatalog interfaces.ResourceCatalog, +) gin.HandlerFunc { baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR") if baseDir == "" { baseDir = "/data/files" @@ -1721,6 +1900,20 @@ func newKBScopedFileServeHandler( c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) return } + if resourceCatalog != nil { + resolvedPath, resource, err := resourceCatalog.ResolvePath(ctx, filePath) + if err != nil { + c.Status(http.StatusNotFound) + return + } + if resource != nil { + if resource.TenantID != ownerTenantID { + c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: resource not accessible"}) + return + } + filePath = resolvedPath + } + } if err := secutils.ValidateKBScopedStoragePath(filePath, ownerTenantID); err != nil { logger.Warnf(ctx, "[Router] /knowledge-bases/:id/files denied path not allowed for KB proxy: owner_tenant_id=%d file_path=%q err=%v", diff --git a/internal/router/router_files_test.go b/internal/router/router_files_test.go index f30f263b..6e586904 100644 --- a/internal/router/router_files_test.go +++ b/internal/router/router_files_test.go @@ -9,6 +9,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/gin-gonic/gin" @@ -22,6 +23,46 @@ type stubFileService struct { getFile func(ctx context.Context, filePath string) (io.ReadCloser, error) } +type stubResourceCatalog struct { + resource *types.StoredResource +} + +func (s *stubResourceCatalog) Register( + context.Context, + uint64, + string, + interfaces.ResourceRegistration, +) (string, error) { + panic("unexpected Register") +} + +func (s *stubResourceCatalog) Resolve(context.Context, string) (*types.StoredResource, error) { + return s.resource, nil +} + +func (s *stubResourceCatalog) ResolvePath(_ context.Context, value string) (string, *types.StoredResource, error) { + if _, ok := types.ParseResourcePath(value); ok && s.resource != nil { + return s.resource.PhysicalPath, s.resource, nil + } + return value, nil, nil +} + +func (s *stubResourceCatalog) Bind(context.Context, string, string, string, string) error { + panic("unexpected Bind") +} + +func (s *stubResourceCatalog) MarkDeleted(context.Context, string) error { + panic("unexpected MarkDeleted") +} + +func (s *stubResourceCatalog) CreateAccessGrant(context.Context, string, time.Duration) (string, error) { + panic("unexpected CreateAccessGrant") +} + +func (s *stubResourceCatalog) ResolveAccessGrant(context.Context, string) (*types.StoredResource, error) { + return s.resource, nil +} + func (s *stubFileService) CheckConnectivity(ctx context.Context) error { return nil } @@ -84,6 +125,87 @@ func TestServeFilesFallsBackToGlobalFileService(t *testing.T) { } } +func TestServeFilesResolvesShortResourceReference(t *testing.T) { + gin.SetMode(gin.TestMode) + t.Setenv("STORAGE_TYPE", "local") + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + const physical = "local://42/exports/a.png" + + engine := gin.New() + var requestedPath string + serveFilesWithResources(engine, &stubFileService{getFile: func(_ context.Context, path string) (io.ReadCloser, error) { + requestedPath = path + return io.NopCloser(strings.NewReader("image")), nil + }}, nil, &stubResourceCatalog{resource: &types.StoredResource{TenantID: 42, PhysicalPath: physical}}) + + req := httptest.NewRequest(http.MethodGet, "/files?file_path="+url.QueryEscape(ref), nil) + req = req.WithContext(context.WithValue(req.Context(), types.TenantInfoContextKey, &types.Tenant{ID: 42})) + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String()) + } + if requestedPath != physical { + t.Fatalf("requested path = %q, want %q", requestedPath, physical) + } +} + +func TestServeFilesRejectsCrossTenantResourceReference(t *testing.T) { + gin.SetMode(gin.TestMode) + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + engine := gin.New() + serveFilesWithResources(engine, &stubFileService{getFile: func(context.Context, string) (io.ReadCloser, error) { + t.Fatal("GetFile should not be called") + return nil, nil + }}, nil, &stubResourceCatalog{resource: &types.StoredResource{TenantID: 7, PhysicalPath: "local://7/exports/a.png"}}) + + req := httptest.NewRequest(http.MethodGet, "/files?file_path="+url.QueryEscape(ref), nil) + req = req.WithContext(context.WithValue(req.Context(), types.TenantInfoContextKey, &types.Tenant{ID: 42})) + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusForbidden) + } +} + +func TestResourceGrantServesShortPublicURL(t *testing.T) { + gin.SetMode(gin.TestMode) + physical := "local://42/exports/a.png" + engine := gin.New() + serveResourceGrants( + engine, + &stubResourceCatalog{resource: &types.StoredResource{ + ID: "resource-1", + TenantID: 42, + PhysicalPath: physical, + OriginalName: "a.png", + MimeType: "image/png", + }}, + &stubTenantService{get: func(_ context.Context, id uint64) (*types.Tenant, error) { + return &types.Tenant{ID: id}, nil + }}, + &stubFileService{getFile: func(_ context.Context, path string) (io.ReadCloser, error) { + if path != physical { + t.Fatalf("path = %q, want %q", path, physical) + } + return io.NopCloser(strings.NewReader("image")), nil + }}, + nil, + ) + + req := httptest.NewRequest(http.MethodGet, "/r/GrantTokenAbCdEfGhIjKlM", nil) + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, req) + if recorder.Code != http.StatusOK || recorder.Body.String() != "image" { + t.Fatalf("status=%d body=%q", recorder.Code, recorder.Body.String()) + } + if got := recorder.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Fatalf("X-Content-Type-Options = %q", got) + } +} + func TestServeFilesDoesNotFallbackWhenProviderDoesNotMatchGlobalStorage(t *testing.T) { gin.SetMode(gin.TestMode) t.Setenv("STORAGE_TYPE", "minio") diff --git a/internal/router/router_static_test.go b/internal/router/router_static_test.go new file mode 100644 index 00000000..f8d7a1a9 --- /dev/null +++ b/internal/router/router_static_test.go @@ -0,0 +1,30 @@ +package router + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestFrontendStaticDoesNotInterceptResourceGrant(t *testing.T) { + gin.SetMode(gin.TestMode) + webDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(webDir, "index.html"), []byte("spa"), 0o600)) + t.Setenv("WEKNORA_WEB_DIR", webDir) + + r := gin.New() + serveFrontendStatic(r) + r.GET("/r/:token", func(c *gin.Context) { c.String(http.StatusOK, "resource") }) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/r/token", nil) + r.ServeHTTP(recorder, request) + + require.Equal(t, http.StatusOK, recorder.Code) + require.Equal(t, "resource", recorder.Body.String()) +} diff --git a/internal/types/interfaces/resource.go b/internal/types/interfaces/resource.go index 07c6cd8e..7d358f08 100644 --- a/internal/types/interfaces/resource.go +++ b/internal/types/interfaces/resource.go @@ -2,18 +2,51 @@ package interfaces import ( "context" + "time" "github.com/Tencent/WeKnora/internal/types" ) -// ResourceCleaner defines the resource cleaner interface +// ResourceCleaner owns process-lifetime cleanup callbacks (database clients, +// worker pools, etc.). It is unrelated to persisted StoredResource records. type ResourceCleaner interface { - // Register registers a resource cleanup function Register(cleanup types.CleanupFunc) - - // RegisterWithName registers a resource cleanup function with a name RegisterWithName(name string, cleanup types.CleanupFunc) - - // Cleanup executes all resource cleanup functions Cleanup(ctx context.Context) []error } + +// ResourceRepository persists stable resource identities, ownership bindings, +// and short-lived access grants. +type ResourceRepository interface { + Create(ctx context.Context, resource *types.StoredResource) error + GetByID(ctx context.Context, id string) (*types.StoredResource, error) + GetByHandle(ctx context.Context, handle string) (*types.StoredResource, error) + GetByTenantLocation(ctx context.Context, tenantID uint64, locationHash string) (*types.StoredResource, error) + MarkDeleted(ctx context.Context, id string) error + CreateBinding(ctx context.Context, binding *types.ResourceBinding) error + CreateGrant(ctx context.Context, grant *types.ResourceAccessGrant) error + GetValidGrant(ctx context.Context, tokenHash string, now time.Time) (*types.ResourceAccessGrant, error) + DeleteExpiredGrants(ctx context.Context, before time.Time) error +} + +// ResourceRegistration describes one physical object at registration time. +type ResourceRegistration struct { + Kind string + MimeType string + OriginalName string + Size int64 + ContentHash string + Temporary bool +} + +// ResourceCatalog maps public resource references to internal storage +// locations and manages their access capabilities. +type ResourceCatalog interface { + Register(ctx context.Context, tenantID uint64, physicalPath string, meta ResourceRegistration) (string, error) + Resolve(ctx context.Context, reference string) (*types.StoredResource, error) + ResolvePath(ctx context.Context, value string) (string, *types.StoredResource, error) + Bind(ctx context.Context, reference, ownerType, ownerID, relation string) error + MarkDeleted(ctx context.Context, reference string) error + CreateAccessGrant(ctx context.Context, reference string, ttl time.Duration) (string, error) + ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) +} diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 27e8f830..d5d27ce0 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -384,7 +384,7 @@ func ParseProviderScheme(filePath string) string { if _, inner, ok := ParseStorageBackendPath(filePath); ok { filePath = inner } - for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs"} { + for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs", "dummy"} { if strings.HasPrefix(filePath, provider+"://") { return provider } diff --git a/internal/types/resource.go b/internal/types/resource.go new file mode 100644 index 00000000..0da9bbfe --- /dev/null +++ b/internal/types/resource.go @@ -0,0 +1,131 @@ +package types + +import ( + "strings" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// Stable resource reference and lifecycle constants. +const ( + ResourceScheme = "resource://" + ResourceHandleLength = 22 + ResourceStateActive = "active" + ResourceStateDeleted = "deleted" + ResourceLifecyclePersistent = "persistent" + ResourceLifecycleTemporary = "temporary" +) + +// StoredResource is the stable application identity of one stored object. PhysicalPath +// is deliberately internal: API responses, persisted rich text and LLM prompts +// use resource:// instead. +type StoredResource struct { + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + Handle string `json:"handle" gorm:"type:varchar(22);not null;uniqueIndex"` + TenantID uint64 `json:"tenant_id" gorm:"not null;index"` + StorageBackendID string `json:"storage_backend_id,omitempty" gorm:"type:varchar(36);index"` + Provider string `json:"provider" gorm:"type:varchar(32);not null"` + PhysicalPath string `json:"-" gorm:"type:text;not null"` + LocationHash string `json:"-" gorm:"type:varchar(64);not null"` + Kind string `json:"kind" gorm:"type:varchar(32);not null;default:'file'"` + MimeType string `json:"mime_type,omitempty" gorm:"type:varchar(255);not null;default:''"` + OriginalName string `json:"original_name,omitempty" gorm:"type:varchar(1024);not null;default:''"` + Size int64 `json:"size" gorm:"not null;default:0"` + ContentHash string `json:"content_hash,omitempty" gorm:"type:varchar(64);not null;default:''"` + Lifecycle string `json:"lifecycle" gorm:"type:varchar(16);not null;default:'persistent'"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + State string `json:"state" gorm:"type:varchar(16);not null;default:'active'"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` +} + +// TableName returns the resource registry table name. +func (StoredResource) TableName() string { return "resources" } + +// BeforeCreate fills resource defaults before persistence. +func (r *StoredResource) BeforeCreate(_ *gorm.DB) error { + if r.ID == "" { + r.ID = uuid.NewString() + } + if r.Kind == "" { + r.Kind = "file" + } + if r.Lifecycle == "" { + r.Lifecycle = ResourceLifecyclePersistent + } + if r.State == "" { + r.State = ResourceStateActive + } + return nil +} + +// ResourceBinding connects a resource to a domain object that owns or uses it. +type ResourceBinding struct { + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + ResourceID string `json:"resource_id" gorm:"type:varchar(36);not null;index"` + TenantID uint64 `json:"tenant_id" gorm:"not null;index"` + OwnerType string `json:"owner_type" gorm:"type:varchar(32);not null"` + OwnerID string `json:"owner_id" gorm:"type:varchar(64);not null"` + Relation string `json:"relation" gorm:"type:varchar(32);not null;default:'attachment'"` + CreatedAt time.Time `json:"created_at"` +} + +// TableName returns the resource binding table name. +func (ResourceBinding) TableName() string { return "resource_bindings" } + +// BeforeCreate assigns an ID before persistence. +func (b *ResourceBinding) BeforeCreate(_ *gorm.DB) error { + if b.ID == "" { + b.ID = uuid.NewString() + } + return nil +} + +// ResourceAccessGrant is a revocable, expiring read capability. +type ResourceAccessGrant struct { + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + TokenHash string `json:"-" gorm:"type:varchar(64);not null;uniqueIndex"` + ResourceID string `json:"resource_id" gorm:"type:varchar(36);not null;index"` + AccessScope string `json:"access_scope" gorm:"type:varchar(16);not null;default:'read'"` + ExpiresAt time.Time `json:"expires_at" gorm:"not null;index"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// TableName returns the resource access grant table name. +func (ResourceAccessGrant) TableName() string { return "resource_access_grants" } + +// BeforeCreate assigns an ID before persistence. +func (g *ResourceAccessGrant) BeforeCreate(_ *gorm.DB) error { + if g.ID == "" { + g.ID = uuid.NewString() + } + return nil +} + +// BuildResourcePath creates a stable application resource reference. +func BuildResourcePath(handle string) string { + return ResourceScheme + strings.TrimSpace(handle) +} + +// ParseResourcePath validates and extracts a canonical resource handle. +func ParseResourcePath(value string) (string, bool) { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, ResourceScheme) { + return "", false + } + handle := strings.TrimPrefix(value, ResourceScheme) + if len(handle) != ResourceHandleLength { + return "", false + } + for _, char := range handle { + if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && + (char < '0' || char > '9') && char != '_' && char != '-' { + return "", false + } + } + return handle, true +} diff --git a/internal/utils/security.go b/internal/utils/security.go index c8b636dc..4267ac00 100644 --- a/internal/utils/security.go +++ b/internal/utils/security.go @@ -164,8 +164,12 @@ func IsValidURL(url string) bool { return false } - // 检查协议, 只允许 http, https, local, minio, cos, tos, oss 协议 - allowedProtocols := []string{"http://", "https://", "local://", "minio://", "cos://", "tos://", "oss://"} + // Internal resource references are resolved through authenticated file + // proxies; provider schemes remain supported for legacy stored content. + allowedProtocols := []string{ + "http://", "https://", "resource://", "storage://", "local://", "minio://", + "cos://", "tos://", "s3://", "oss://", "ks3://", "obs://", + } isAllowed := false for _, protocol := range allowedProtocols { if strings.HasPrefix(strings.ToLower(url), protocol) { @@ -486,6 +490,9 @@ func IsValidImageURL(url string) bool { if !IsValidURL(url) { return false } + if strings.HasPrefix(strings.ToLower(url), "resource://") { + return true + } // 检查是否为图片文件 imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"} diff --git a/migrations/sqlite/000000_init.down.sql b/migrations/sqlite/000000_init.down.sql index ec84c423..a402f229 100644 --- a/migrations/sqlite/000000_init.down.sql +++ b/migrations/sqlite/000000_init.down.sql @@ -22,5 +22,8 @@ DROP TABLE IF EXISTS sessions; DROP TABLE IF EXISTS knowledges; DROP TABLE IF EXISTS knowledge_bases; DROP TABLE IF EXISTS storage_backends; +DROP TABLE IF EXISTS resource_access_grants; +DROP TABLE IF EXISTS resource_bindings; +DROP TABLE IF EXISTS resources; DROP TABLE IF EXISTS models; DROP TABLE IF EXISTS tenants; diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index 46783b81..2d7bc062 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -827,6 +827,59 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = 1; CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id); +CREATE TABLE IF NOT EXISTS resources ( + id VARCHAR(36) PRIMARY KEY, + handle VARCHAR(22) NOT NULL UNIQUE, + tenant_id INTEGER NOT NULL, + storage_backend_id VARCHAR(36), + provider VARCHAR(32) NOT NULL, + physical_path TEXT NOT NULL, + location_hash VARCHAR(64) NOT NULL, + kind VARCHAR(32) NOT NULL DEFAULT 'file', + mime_type VARCHAR(255) NOT NULL DEFAULT '', + original_name VARCHAR(1024) NOT NULL DEFAULT '', + size INTEGER NOT NULL DEFAULT 0, + content_hash VARCHAR(64) NOT NULL DEFAULT '', + lifecycle VARCHAR(16) NOT NULL DEFAULT 'persistent', + expires_at DATETIME, + state VARCHAR(16) NOT NULL DEFAULT 'active', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + deleted_at DATETIME +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_resources_tenant_location + ON resources(tenant_id, location_hash) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_resources_tenant ON resources(tenant_id); +CREATE INDEX IF NOT EXISTS idx_resources_backend ON resources(storage_backend_id); + +CREATE TABLE IF NOT EXISTS resource_bindings ( + id VARCHAR(36) PRIMARY KEY, + resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE, + tenant_id INTEGER NOT NULL, + owner_type VARCHAR(32) NOT NULL, + owner_id VARCHAR(64) NOT NULL, + relation VARCHAR(32) NOT NULL DEFAULT 'attachment', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_resource_bindings_unique + ON resource_bindings(resource_id, owner_type, owner_id, relation); +CREATE INDEX IF NOT EXISTS idx_resource_bindings_owner + ON resource_bindings(tenant_id, owner_type, owner_id); + +CREATE TABLE IF NOT EXISTS resource_access_grants ( + id VARCHAR(36) PRIMARY KEY, + token_hash VARCHAR(64) NOT NULL UNIQUE, + resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE, + access_scope VARCHAR(16) NOT NULL DEFAULT 'read', + expires_at DATETIME NOT NULL, + revoked_at DATETIME, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_resource_access_grants_resource + ON resource_access_grants(resource_id); +CREATE INDEX IF NOT EXISTS idx_resource_access_grants_expires + ON resource_access_grants(expires_at); + CREATE TABLE IF NOT EXISTS tenant_api_keys ( id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id INTEGER NOT NULL, diff --git a/migrations/versioned/000069_resource_registry.down.sql b/migrations/versioned/000069_resource_registry.down.sql new file mode 100644 index 00000000..0b909bb0 --- /dev/null +++ b/migrations/versioned/000069_resource_registry.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS resource_access_grants; +DROP TABLE IF EXISTS resource_bindings; +DROP TABLE IF EXISTS resources; diff --git a/migrations/versioned/000069_resource_registry.up.sql b/migrations/versioned/000069_resource_registry.up.sql new file mode 100644 index 00000000..f8297223 --- /dev/null +++ b/migrations/versioned/000069_resource_registry.up.sql @@ -0,0 +1,55 @@ +CREATE TABLE IF NOT EXISTS resources ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + handle VARCHAR(22) NOT NULL UNIQUE, + tenant_id BIGINT NOT NULL, + storage_backend_id VARCHAR(36), + provider VARCHAR(32) NOT NULL, + physical_path TEXT NOT NULL, + location_hash VARCHAR(64) NOT NULL, + kind VARCHAR(32) NOT NULL DEFAULT 'file', + mime_type VARCHAR(255) NOT NULL DEFAULT '', + original_name VARCHAR(1024) NOT NULL DEFAULT '', + size BIGINT NOT NULL DEFAULT 0, + content_hash VARCHAR(64) NOT NULL DEFAULT '', + lifecycle VARCHAR(16) NOT NULL DEFAULT 'persistent', + expires_at TIMESTAMP NULL, + state VARCHAR(16) NOT NULL DEFAULT 'active', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_resources_tenant_location + ON resources(tenant_id, location_hash) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_resources_tenant ON resources(tenant_id); +CREATE INDEX IF NOT EXISTS idx_resources_backend ON resources(storage_backend_id); + +CREATE TABLE IF NOT EXISTS resource_bindings ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE, + tenant_id BIGINT NOT NULL, + owner_type VARCHAR(32) NOT NULL, + owner_id VARCHAR(64) NOT NULL, + relation VARCHAR(32) NOT NULL DEFAULT 'attachment', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_resource_bindings_unique + ON resource_bindings(resource_id, owner_type, owner_id, relation); +CREATE INDEX IF NOT EXISTS idx_resource_bindings_owner + ON resource_bindings(tenant_id, owner_type, owner_id); + +CREATE TABLE IF NOT EXISTS resource_access_grants ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + token_hash VARCHAR(64) NOT NULL UNIQUE, + resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE, + access_scope VARCHAR(16) NOT NULL DEFAULT 'read', + expires_at TIMESTAMP NOT NULL, + revoked_at TIMESTAMP NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_resource_access_grants_resource + ON resource_access_grants(resource_id); +CREATE INDEX IF NOT EXISTS idx_resource_access_grants_expires + ON resource_access_grants(expires_at);