feat(resource): add stable resource registry with LLM-context alias compaction

Introduce resource://<handle> as the stable application identity for stored
objects, decoupling API responses, persisted rich text, and LLM prompts from
physical storage paths. Adds a resource catalog service (register/resolve/bind,
revocable expiring access grants), a request-local alias registry that compacts
resource references while in the LLM context and restores them (including across
streaming chunks), plus router, IM, chat-pipeline, and multimodal integration.

Includes schema migrations (resources, resource_bindings, resource_access_grants)
for sqlite, and hardens URL/output security handling.
This commit is contained in:
wizardchen
2026-07-15 16:31:21 +08:00
committed by lyingbug
parent 9733bd28bf
commit 6d14cc3536
32 changed files with 1790 additions and 59 deletions
+1 -1
View File
@@ -2,7 +2,7 @@ export const domPurifyForbidTags = ['script', 'style', 'object', 'embed', 'form'
export const domPurifyForbidAttr = ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'] as const;
export const domPurifyAllowedUriRegexp =
/^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|(?:storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
/^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|(?:resource|storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i;
/** Shared DOMPurify security options (FORBID_*, URI scheme, DOM flags). */
export const domPurifySecurityOptions = {
+12
View File
@@ -24,3 +24,15 @@ test('protectProviderImageSrcInHTML uses a placeholder src for storage-backend i
/data-protected-src="storage:\/\/c0d93536-702c-4977-aa5e-fe670073c3cb\/local:\/\/10000\/exports\/a\.png"/,
)
})
test('protectProviderImageSrcInHTML uses a placeholder src for resource references', () => {
const html = '<p><img alt="preview" src="resource://AbCdEfGhIjKlMnOpQrStUv"></p>'
const sanitized = protectProviderImageSrcInHTML(html)
const renderedSrc = sanitized.match(/<img[^>]*\ssrc="([^"]+)"/)?.[1]
assert.match(renderedSrc || '', /^data:image\/gif;base64,/)
assert.match(
sanitized,
/data-protected-src="resource:\/\/AbCdEfGhIjKlMnOpQrStUv"/,
)
})
+2 -2
View File
@@ -12,7 +12,7 @@ import {
} from './markdownDomPurify.ts';
const PROVIDER_IMAGE_PLACEHOLDER = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==';
const PROVIDER_SCHEME_PATTERN = 'local|minio|cos|tos|s3|oss|ks3|obs';
const PROVIDER_SCHEME_PATTERN = 'resource|local|minio|cos|tos|s3|oss|ks3|obs';
const PROVIDER_FILE_SCHEME_RE = new RegExp(`^(${PROVIDER_SCHEME_PATTERN}):\\/\\/\\S+$`, 'i');
const STORAGE_BACKEND_FILE_SCHEME_RE = new RegExp(
`^storage:\\/\\/[0-9A-Za-z_-]+\\/(${PROVIDER_SCHEME_PATTERN}):\\/\\/\\S+$`,
@@ -499,7 +499,7 @@ export async function hydrateProtectedFileImages(
}
const images = root.querySelectorAll<HTMLImageElement>(
'img[data-protected-src], img[src^="storage://"], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"], img[src^="obs://"]',
'img[data-protected-src], img[src^="resource://"], img[src^="storage://"], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"], img[src^="ks3://"], img[src^="obs://"]',
);
if (!images.length) {
return;
+3
View File
@@ -14,6 +14,7 @@ import (
"github.com/Tencent/WeKnora/internal/common"
appconfig "github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/llmresource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
@@ -49,6 +50,7 @@ type AgentEngine struct {
memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional)
lastUsage types.TokenUsage // Token usage from the most recent LLM call
lastSentMsgCount int // Number of messages sent in the most recent LLM call
resourceRefs *llmresource.Registry // request-local aliases for durable resource references
}
// ImageDescriberFunc generates a text description of an image.
@@ -83,6 +85,7 @@ func NewAgentEngine(
sessionID: sessionID,
systemPromptTemplate: systemPromptTemplate,
tokenEstimator: tokenEst,
resourceRefs: llmresource.NewRegistry(),
}
// Initialize memory consolidator if context window management is configured
+30 -3
View File
@@ -24,16 +24,22 @@ type mockResponse struct {
type mockChat struct {
mu sync.Mutex
responses []mockResponse
calls [][]chat.Message
callCount int
}
func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (<-chan types.StreamResponse, error) {
func (m *mockChat) ChatStream(
_ context.Context,
messages []chat.Message,
_ *chat.ChatOptions,
) (<-chan types.StreamResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.callCount >= len(m.responses) {
return nil, fmt.Errorf("unexpected ChatStream call #%d (only %d responses prepared)", m.callCount, len(m.responses))
}
resp := m.responses[m.callCount]
m.calls = append(m.calls, append([]chat.Message(nil), messages...))
m.callCount++
ch := make(chan types.StreamResponse, len(resp.chunks))
@@ -44,6 +50,25 @@ func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatO
return ch, nil
}
func TestStreamLLMResourceAliasesRoundTrip(t *testing.T) {
const ref = "resource://AbCdEfGhIjKlMnOpQrStUv"
model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{
{ResponseType: types.ResponseTypeAnswer, Content: "![image](res:0"},
{ResponseType: types.ResponseTypeAnswer, Content: "001)", Done: true},
}}}}
engine := newTestEngine(t, model)
result, err := engine.streamLLMToEventBus(
context.Background(),
[]chat.Message{{Role: "tool", Content: "source=" + ref}},
nil,
nil,
)
require.NoError(t, err)
require.Equal(t, "![image]("+ref+")", result.Content)
require.Len(t, model.calls, 1)
require.Equal(t, "source=res:0001", model.calls[0][0].Content)
}
func (m *mockChat) Chat(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (*types.ChatResponse, error) {
return nil, fmt.Errorf("not implemented")
}
@@ -270,8 +295,10 @@ func TestStreamThinkingToEventBus_SplitsInlineThinkBlock(t *testing.T) {
mock := &mockChat{
responses: []mockResponse{
{chunks: []types.StreamResponse{
{ResponseType: types.ResponseTypeAnswer, Content: "<think>hidden reasoning</think>Visible answer.",
Done: true, FinishReason: "stop"},
{
ResponseType: types.ResponseTypeAnswer, Content: "<think>hidden reasoning</think>Visible answer.",
Done: true, FinishReason: "stop",
},
}},
},
}
+16
View File
@@ -9,6 +9,7 @@ import (
agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/llmresource"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
@@ -37,6 +38,7 @@ func (e *AgentEngine) streamLLMToEventBus(
llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout())
defer llmCancel()
messages = e.resourceRefs.EncodeMessages(messages)
stream, err := e.chatModel.ChatStream(llmCtx, messages, opts)
if err != nil {
logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err)
@@ -47,6 +49,8 @@ func (e *AgentEngine) streamLLMToEventBus(
chunkCount := 0
responseTypeCounts := make(map[string]int)
firstChunkTime := time.Time{}
answerDecoder := llmresource.NewStreamDecoder(e.resourceRefs)
thinkingDecoder := llmresource.NewStreamDecoder(e.resourceRefs)
for chunk := range stream {
chunkCount++
@@ -62,6 +66,12 @@ func (e *AgentEngine) streamLLMToEventBus(
result.StreamError = chunk.Content
continue
}
if chunk.ResponseType == types.ResponseTypeThinking {
chunk.Content = thinkingDecoder.Feed(chunk.Content)
} else {
chunk.Content = answerDecoder.Feed(chunk.Content)
}
e.resourceRefs.DecodeToolCalls(chunk.ToolCalls)
if chunk.Content != "" {
isExtracted := chunk.Data != nil && chunk.Data["source"] != nil
@@ -90,6 +100,12 @@ func (e *AgentEngine) streamLLMToEventBus(
emitFunc(&chunk, result.Content)
}
}
result.Content += answerDecoder.Flush()
result.ReasoningContent += thinkingDecoder.Flush()
// Some providers stream tool-call argument fragments but expose the
// accumulated call in the final chunk. Decode once more after assembly so
// aliases split across provider chunks cannot leak into tool execution.
e.resourceRefs.DecodeToolCalls(result.ToolCalls)
// Stream diagnostic summary: helps identify non-streaming patterns
streamDuration := time.Duration(0)
@@ -0,0 +1,97 @@
package repository
import (
"context"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type resourceRepository struct{ db *gorm.DB }
// NewResourceRepository creates the persistence adapter for resource metadata.
func NewResourceRepository(db *gorm.DB) interfaces.ResourceRepository {
return &resourceRepository{db: db}
}
func (r *resourceRepository) Create(ctx context.Context, resource *types.StoredResource) error {
return r.db.WithContext(ctx).Create(resource).Error
}
func (r *resourceRepository) GetByID(ctx context.Context, id string) (*types.StoredResource, error) {
var resource types.StoredResource
err := r.db.WithContext(ctx).Where("id = ? AND state = ?", id, types.ResourceStateActive).First(&resource).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &resource, err
}
func (r *resourceRepository) GetByHandle(ctx context.Context, handle string) (*types.StoredResource, error) {
var resource types.StoredResource
err := r.db.WithContext(ctx).
Where("handle = ? AND state = ?", handle, types.ResourceStateActive).
First(&resource).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &resource, err
}
func (r *resourceRepository) GetByTenantLocation(
ctx context.Context,
tenantID uint64,
locationHash string,
) (*types.StoredResource, error) {
var resource types.StoredResource
err := r.db.WithContext(ctx).
Where(
"tenant_id = ? AND location_hash = ? AND state = ?",
tenantID,
locationHash,
types.ResourceStateActive,
).
First(&resource).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &resource, err
}
func (r *resourceRepository) MarkDeleted(ctx context.Context, id string) error {
return r.db.WithContext(ctx).Model(&types.StoredResource{}).Where("id = ?", id).
Updates(map[string]interface{}{"state": types.ResourceStateDeleted, "deleted_at": time.Now()}).Error
}
func (r *resourceRepository) CreateBinding(ctx context.Context, binding *types.ResourceBinding) error {
return r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(binding).Error
}
func (r *resourceRepository) CreateGrant(ctx context.Context, grant *types.ResourceAccessGrant) error {
return r.db.WithContext(ctx).Create(grant).Error
}
func (r *resourceRepository) GetValidGrant(
ctx context.Context,
tokenHash string,
now time.Time,
) (*types.ResourceAccessGrant, error) {
var grant types.ResourceAccessGrant
err := r.db.WithContext(ctx).
Where("token_hash = ? AND revoked_at IS NULL AND expires_at > ?", tokenHash, now).
First(&grant).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &grant, err
}
func (r *resourceRepository) DeleteExpiredGrants(ctx context.Context, before time.Time) error {
return r.db.WithContext(ctx).
Where("expires_at <= ? OR revoked_at IS NOT NULL", before).
Delete(&types.ResourceAccessGrant{}).Error
}
@@ -3,6 +3,7 @@ package chatpipeline
import (
"context"
"github.com/Tencent/WeKnora/internal/llmresource"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -51,6 +52,8 @@ func (p *PluginChatCompletion) OnEvent(
"message_count": len(chatManage.History) + 2,
})
chatMessages := prepareMessagesWithHistory(chatManage)
resourceRefs := llmresource.NewRegistry()
chatMessages = resourceRefs.EncodeMessages(chatMessages)
// Call the chat model to generate response
pipelineInfo(ctx, "Completion", "model_call", map[string]interface{}{
@@ -64,6 +67,7 @@ func (p *PluginChatCompletion) OnEvent(
})
return ErrModelCall.WithError(err)
}
resourceRefs.DecodeResponse(chatResponse)
pipelineInfo(ctx, "Completion", "output", map[string]interface{}{
"answer_preview": chatResponse.Content,
@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/llmresource"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -55,6 +56,8 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
// Prepare base messages without history
chatMessages := prepareMessagesWithHistory(chatManage)
resourceRefs := llmresource.NewRegistry()
chatMessages = resourceRefs.EncodeMessages(chatMessages)
pipelineInfo(ctx, "Stream", "messages_ready", map[string]interface{}{
"message_count": len(chatMessages),
"system_prompt": chatMessages[0].Content,
@@ -105,6 +108,8 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
// The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled
// and the upstream channel is not closed promptly.
go func() {
answerDecoder := llmresource.NewStreamDecoder(resourceRefs)
thinkingDecoder := llmresource.NewStreamDecoder(resourceRefs)
thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8])
answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8])
thinkingOpen := false
@@ -135,6 +140,22 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
case response, ok := <-responseChan:
if !ok {
if tail := thinkingDecoder.Flush(); tail != "" {
_ = eventBus.Emit(ctx, types.Event{
ID: thinkingID,
Type: types.EventType(event.EventAgentThought),
SessionID: chatManage.SessionID,
Data: event.AgentThoughtData{Content: tail},
})
}
if tail := answerDecoder.Flush(); tail != "" {
_ = eventBus.Emit(ctx, types.Event{
ID: answerID,
Type: types.EventType(event.EventAgentFinalAnswer),
SessionID: chatManage.SessionID,
Data: event.AgentFinalAnswerData{Content: tail},
})
}
closeThinking()
pipelineInfo(ctx, "Stream", "channel_close", map[string]interface{}{
"session_id": chatManage.SessionID,
@@ -161,6 +182,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
}
if response.ResponseType == types.ResponseTypeThinking {
response.Content = thinkingDecoder.Feed(response.Content)
if response.Content != "" {
thinkingOpen = true
eventBus.Emit(ctx, types.Event{
@@ -180,6 +202,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
}
if response.ResponseType == types.ResponseTypeAnswer {
response.Content = answerDecoder.Feed(response.Content)
closeThinking()
eventBus.Emit(ctx, types.Event{
ID: answerID,
+3 -2
View File
@@ -3,6 +3,7 @@ package file
import (
"context"
"errors"
"fmt"
"io"
"mime/multipart"
@@ -30,7 +31,7 @@ func NewDummyFileService() interfaces.FileService {
func (s *DummyFileService) SaveFile(ctx context.Context,
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
) (string, error) {
return uuid.New().String(), nil
return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil
}
// GetFile always returns an error as dummy service doesn't store files
@@ -45,7 +46,7 @@ func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) erro
// SaveBytes pretends to save bytes but just returns a random UUID
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
return uuid.New().String(), nil
return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil
}
// CopyFile is a no-op for the dummy service: it logs a warning and returns the
@@ -0,0 +1,193 @@
package file
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"mime"
"mime/multipart"
"os"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// resourceCatalogFileService keeps provider drivers physical-path-only while
// exposing stable resource:// references to the application layer.
type resourceCatalogFileService struct {
inner interfaces.FileService
catalog interfaces.ResourceCatalog
externalURL string
}
// NewResourceCatalogFileService decorates a physical FileService with stable
// resource registration and resolution.
func NewResourceCatalogFileService(
inner interfaces.FileService,
catalog interfaces.ResourceCatalog,
) interfaces.FileService {
if inner == nil || catalog == nil {
return inner
}
return &resourceCatalogFileService{
inner: inner,
catalog: catalog,
externalURL: strings.TrimRight(strings.TrimSpace(os.Getenv("APP_EXTERNAL_URL")), "/"),
}
}
func (s *resourceCatalogFileService) CheckConnectivity(ctx context.Context) error {
return s.inner.CheckConnectivity(ctx)
}
func resourceKind(name string) (string, string) {
mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name)))
kind := "file"
switch {
case strings.HasPrefix(mimeType, "image/"):
kind = "image"
case strings.HasPrefix(mimeType, "audio/"):
kind = "audio"
case strings.HasPrefix(mimeType, "video/"):
kind = "video"
}
return kind, mimeType
}
func (s *resourceCatalogFileService) register(
ctx context.Context,
physical string,
tenantID uint64,
name string,
size int64,
temporary bool,
contentHash string,
) (string, error) {
kind, mimeType := resourceKind(name)
ref, err := s.catalog.Register(ctx, tenantID, physical, interfaces.ResourceRegistration{
Kind: kind,
MimeType: mimeType,
OriginalName: filepath.Base(name),
Size: size,
ContentHash: contentHash,
Temporary: temporary,
})
if err != nil {
_ = s.inner.DeleteFile(ctx, physical)
return "", fmt.Errorf("register stored resource: %w", err)
}
return ref, nil
}
func (s *resourceCatalogFileService) SaveFile(
ctx context.Context,
file *multipart.FileHeader,
tenantID uint64,
knowledgeID string,
) (string, error) {
physical, err := s.inner.SaveFile(ctx, file, tenantID, knowledgeID)
if err != nil {
return "", err
}
ref, err := s.register(ctx, physical, tenantID, file.Filename, file.Size, false, "")
if err != nil {
return "", err
}
if knowledgeID != "" {
if err := s.catalog.Bind(ctx, ref, "knowledge", knowledgeID, "source_file"); err != nil {
_ = s.DeleteFile(ctx, ref)
return "", fmt.Errorf("bind stored resource: %w", err)
}
}
return ref, nil
}
func (s *resourceCatalogFileService) SaveBytes(
ctx context.Context,
data []byte,
tenantID uint64,
fileName string,
temp bool,
) (string, error) {
physical, err := s.inner.SaveBytes(ctx, data, tenantID, fileName, temp)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return s.register(ctx, physical, tenantID, fileName, int64(len(data)), temp, hex.EncodeToString(sum[:]))
}
func (s *resourceCatalogFileService) resolve(ctx context.Context, value string) (string, bool, error) {
physical, resource, err := s.catalog.ResolvePath(ctx, value)
return physical, resource != nil, err
}
func (s *resourceCatalogFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
physical, _, err := s.resolve(ctx, filePath)
if err != nil {
return nil, err
}
return s.inner.GetFile(ctx, physical)
}
func (s *resourceCatalogFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
physical, isResource, err := s.resolve(ctx, filePath)
if err != nil {
return "", err
}
if isResource && s.externalURL != "" {
token, grantErr := s.catalog.CreateAccessGrant(ctx, filePath, 2*time.Hour)
if grantErr != nil {
return "", grantErr
}
return s.externalURL + "/r/" + token, nil
}
return s.inner.GetFileURL(ctx, physical)
}
func (s *resourceCatalogFileService) DeleteFile(ctx context.Context, filePath string) error {
physical, isResource, err := s.resolve(ctx, filePath)
if err != nil {
return err
}
if err := s.inner.DeleteFile(ctx, physical); err != nil {
return err
}
if isResource {
return s.catalog.MarkDeleted(ctx, filePath)
}
return nil
}
func (s *resourceCatalogFileService) CopyFile(
ctx context.Context,
filePath string,
tenantID uint64,
knowledgeID string,
) (string, error) {
physical, _, err := s.resolve(ctx, filePath)
if err != nil {
return "", err
}
copied, err := s.inner.CopyFile(ctx, physical, tenantID, knowledgeID)
if err != nil {
return "", err
}
ref, err := s.register(ctx, copied, tenantID, filepath.Base(physical), 0, false, "")
if err != nil {
return "", err
}
if knowledgeID != "" {
if err := s.catalog.Bind(ctx, ref, "knowledge", knowledgeID, "source_file"); err != nil {
_ = s.DeleteFile(ctx, ref)
return "", fmt.Errorf("bind copied resource: %w", err)
}
}
return ref, nil
}
var _ interfaces.FileService = (*resourceCatalogFileService)(nil)
@@ -0,0 +1,107 @@
package file
import (
"context"
"io"
"mime/multipart"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
)
type catalogStub struct {
resource *types.StoredResource
ref string
}
func (c *catalogStub) Register(
_ context.Context,
tenantID uint64,
physicalPath string,
meta interfaces.ResourceRegistration,
) (string, error) {
c.resource = &types.StoredResource{
ID: "resource-1",
Handle: "AbCdEfGhIjKlMnOpQrStUv",
TenantID: tenantID,
PhysicalPath: physicalPath,
OriginalName: meta.OriginalName,
}
c.ref = types.BuildResourcePath(c.resource.Handle)
return c.ref, nil
}
func (c *catalogStub) Resolve(_ context.Context, _ string) (*types.StoredResource, error) {
return c.resource, nil
}
func (c *catalogStub) ResolvePath(_ context.Context, value string) (string, *types.StoredResource, error) {
if value == c.ref {
return c.resource.PhysicalPath, c.resource, nil
}
return value, nil, nil
}
func (c *catalogStub) Bind(context.Context, string, string, string, string) error { return nil }
func (c *catalogStub) MarkDeleted(context.Context, string) error { return nil }
func (c *catalogStub) CreateAccessGrant(context.Context, string, time.Duration) (string, error) {
return "GrantTokenAbCdEfGhIjKl", nil
}
func (c *catalogStub) ResolveAccessGrant(context.Context, string) (*types.StoredResource, error) {
return c.resource, nil
}
type physicalFileStub struct {
savedPath string
readPath string
}
func (s *physicalFileStub) CheckConnectivity(context.Context) error { return nil }
func (s *physicalFileStub) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {
return "", nil
}
func (s *physicalFileStub) SaveBytes(context.Context, []byte, uint64, string, bool) (string, error) {
return s.savedPath, nil
}
func (s *physicalFileStub) GetFile(_ context.Context, path string) (io.ReadCloser, error) {
s.readPath = path
return io.NopCloser(strings.NewReader("body")), nil
}
func (s *physicalFileStub) GetFileURL(context.Context, string) (string, error) { return "", nil }
func (s *physicalFileStub) DeleteFile(context.Context, string) error { return nil }
func (s *physicalFileStub) CopyFile(context.Context, string, uint64, string) (string, error) {
return "", nil
}
func TestResourceCatalogFileServiceReturnsReferenceAndResolvesReads(t *testing.T) {
inner := &physicalFileStub{savedPath: "local://7/exports/a.png"}
catalog := &catalogStub{}
svc := NewResourceCatalogFileService(inner, catalog)
ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false)
require.NoError(t, err)
require.Equal(t, "resource://AbCdEfGhIjKlMnOpQrStUv", ref)
reader, err := svc.GetFile(context.Background(), ref)
require.NoError(t, err)
require.NoError(t, reader.Close())
require.Equal(t, inner.savedPath, inner.readPath)
}
func TestResourceCatalogFileServiceReturnsShortExternalGrantURL(t *testing.T) {
t.Setenv("APP_EXTERNAL_URL", "https://weknora.example.com/")
inner := &physicalFileStub{savedPath: "local://7/exports/a.png"}
catalog := &catalogStub{}
svc := NewResourceCatalogFileService(inner, catalog)
ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false)
require.NoError(t, err)
externalURL, err := svc.GetFileURL(context.Background(), ref)
require.NoError(t, err)
require.Equal(t, "https://weknora.example.com/r/GrantTokenAbCdEfGhIjKl", externalURL)
}
@@ -80,6 +80,10 @@ type ImageMultimodalService struct {
// fallback in knowledgeService.resolveFileService.
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
// resourceCatalog resolves resource:// references to their owning storage
// backend so multimodal reads target the resource's real backend instead of
// the knowledge base's currently configured one.
resourceCatalog interfaces.ResourceCatalog
// spanTracker records this image's subspan under the parent attempt's
// multimodal stage. nil-safe — falls back to no-op via tracker().
@@ -99,6 +103,7 @@ func NewImageMultimodalService(
redisClient *redis.Client,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalog interfaces.ResourceCatalog,
spanTracker SpanTracker,
) interfaces.TaskHandler {
return &ImageMultimodalService{
@@ -114,6 +119,7 @@ func NewImageMultimodalService(
redisClient: redisClient,
fileSvc: fileSvc,
storageResolver: storageResolver,
resourceCatalog: resourceCatalog,
spanTracker: spanTracker,
}
}
@@ -554,6 +560,18 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
backendID, _, _ := types.ParseStorageBackendPath(payload.ImageURL)
provider := types.ParseProviderScheme(payload.ImageURL)
// A resource:// reference carries no provider/backend in the URL itself; the
// authoritative backend lives on the stored resource record. Using the KB's
// currently configured backend here would break reads when the resource was
// stored on a different backend (multi-backend / post-migration).
if _, isResourceRef := types.ParseResourcePath(payload.ImageURL); isResourceRef && s.resourceCatalog != nil {
if resource, resErr := s.resourceCatalog.Resolve(ctx, payload.ImageURL); resErr != nil {
logger.Warnf(ctx, "[ImageMultimodal] resolve resource reference failed: url=%s err=%v", payload.ImageURL, resErr)
} else if resource != nil {
backendID = resource.StorageBackendID
provider = strings.ToLower(strings.TrimSpace(resource.Provider))
}
}
if provider == "" {
kb, kbErr := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if kbErr != nil {
@@ -590,7 +608,8 @@ func (s *ImageMultimodalService) resolveFileServiceForPayload(ctx context.Contex
// file before falling back to the URL.
// - For plain http(s):// URLs it uses the SSRF-safe downloader.
func (s *ImageMultimodalService) readImageBytes(ctx context.Context, payload types.ImageMultimodalPayload) ([]byte, error) {
if types.ParseProviderScheme(payload.ImageURL) != "" {
_, isResourceRef := types.ParseResourcePath(payload.ImageURL)
if isResourceRef || types.ParseProviderScheme(payload.ImageURL) != "" {
fileSvc := s.resolveFileServiceForPayload(ctx, payload)
if fileSvc == nil {
return nil, fmt.Errorf("no file service available for %s", payload.ImageURL)
+209
View File
@@ -0,0 +1,209 @@
package service
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
const defaultResourceGrantTTL = 2 * time.Hour
type resourceCatalog struct {
repo interfaces.ResourceRepository
}
// NewResourceCatalog creates the stable resource-reference domain service.
func NewResourceCatalog(repo interfaces.ResourceRepository) interfaces.ResourceCatalog {
return &resourceCatalog{repo: repo}
}
func randomResourceToken() (string, error) {
buf := make([]byte, 16)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return base64.RawURLEncoding.EncodeToString(buf), nil
}
func resourceLocationHash(path string) string {
sum := sha256.Sum256([]byte(path))
return hex.EncodeToString(sum[:])
}
func (s *resourceCatalog) Register(
ctx context.Context,
tenantID uint64,
physicalPath string,
meta interfaces.ResourceRegistration,
) (string, error) {
physicalPath = strings.TrimSpace(physicalPath)
if tenantID == 0 || physicalPath == "" {
return "", fmt.Errorf("resource registration requires tenant and physical path")
}
if _, ok := types.ParseResourcePath(physicalPath); ok {
return physicalPath, nil
}
locationHash := resourceLocationHash(physicalPath)
existing, err := s.repo.GetByTenantLocation(ctx, tenantID, locationHash)
if err != nil {
return "", err
}
if existing != nil {
return types.BuildResourcePath(existing.Handle), nil
}
backendID, inner, scoped := types.ParseStorageBackendPath(physicalPath)
providerPath := physicalPath
if scoped {
providerPath = inner
}
provider := types.ParseProviderScheme(providerPath)
if provider == "" {
return "", fmt.Errorf("resource physical path has unsupported provider scheme")
}
lifecycle := types.ResourceLifecyclePersistent
if meta.Temporary {
lifecycle = types.ResourceLifecycleTemporary
}
for attempt := 0; attempt < 4; attempt++ {
handle, tokenErr := randomResourceToken()
if tokenErr != nil {
return "", tokenErr
}
resource := &types.StoredResource{
Handle: handle,
TenantID: tenantID,
StorageBackendID: backendID,
Provider: provider,
PhysicalPath: physicalPath,
LocationHash: locationHash,
Kind: meta.Kind,
MimeType: meta.MimeType,
OriginalName: meta.OriginalName,
Size: meta.Size,
ContentHash: meta.ContentHash,
Lifecycle: lifecycle,
}
if err := s.repo.Create(ctx, resource); err == nil {
return types.BuildResourcePath(handle), nil
} else if !strings.Contains(strings.ToLower(err.Error()), "unique") {
return "", err
}
existing, lookupErr := s.repo.GetByTenantLocation(ctx, tenantID, locationHash)
if lookupErr == nil && existing != nil {
return types.BuildResourcePath(existing.Handle), nil
}
}
return "", fmt.Errorf("failed to allocate unique resource handle")
}
func (s *resourceCatalog) Resolve(ctx context.Context, reference string) (*types.StoredResource, error) {
handle, ok := types.ParseResourcePath(reference)
if !ok {
return nil, fmt.Errorf("invalid resource reference")
}
resource, err := s.repo.GetByHandle(ctx, handle)
if err != nil {
return nil, err
}
if resource == nil {
return nil, fmt.Errorf("resource not found")
}
return resource, nil
}
func (s *resourceCatalog) ResolvePath(ctx context.Context, value string) (string, *types.StoredResource, error) {
if _, ok := types.ParseResourcePath(value); !ok {
return value, nil, nil
}
resource, err := s.Resolve(ctx, value)
if err != nil {
return "", nil, err
}
return resource.PhysicalPath, resource, nil
}
func (s *resourceCatalog) Bind(ctx context.Context, reference, ownerType, ownerID, relation string) error {
resource, err := s.Resolve(ctx, reference)
if err != nil {
return err
}
if strings.TrimSpace(ownerType) == "" || strings.TrimSpace(ownerID) == "" {
return fmt.Errorf("resource binding requires owner type and id")
}
if relation == "" {
relation = "attachment"
}
return s.repo.CreateBinding(ctx, &types.ResourceBinding{
ResourceID: resource.ID,
TenantID: resource.TenantID,
OwnerType: ownerType,
OwnerID: ownerID,
Relation: relation,
})
}
func (s *resourceCatalog) MarkDeleted(ctx context.Context, reference string) error {
resource, err := s.Resolve(ctx, reference)
if err != nil {
return err
}
return s.repo.MarkDeleted(ctx, resource.ID)
}
func (s *resourceCatalog) CreateAccessGrant(ctx context.Context, reference string, ttl time.Duration) (string, error) {
resource, err := s.Resolve(ctx, reference)
if err != nil {
return "", err
}
if ttl <= 0 {
ttl = defaultResourceGrantTTL
}
// Opportunistic cleanup keeps high-volume IM rendering from accumulating
// expired capability rows; failure is non-fatal to the current grant.
_ = s.repo.DeleteExpiredGrants(ctx, time.Now())
for attempt := 0; attempt < 4; attempt++ {
token, tokenErr := randomResourceToken()
if tokenErr != nil {
return "", tokenErr
}
grant := &types.ResourceAccessGrant{
TokenHash: resourceLocationHash(token),
ResourceID: resource.ID,
AccessScope: "read",
ExpiresAt: time.Now().Add(ttl),
}
if err := s.repo.CreateGrant(ctx, grant); err == nil {
return token, nil
} else if !strings.Contains(strings.ToLower(err.Error()), "unique") {
return "", err
}
}
return "", fmt.Errorf("failed to allocate unique resource access token")
}
func (s *resourceCatalog) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) {
grant, err := s.repo.GetValidGrant(ctx, resourceLocationHash(strings.TrimSpace(token)), time.Now())
if err != nil {
return nil, err
}
if grant == nil {
return nil, fmt.Errorf("resource access grant is invalid or expired")
}
resource, err := s.repo.GetByID(ctx, grant.ResourceID)
if err != nil {
return nil, err
}
if resource == nil {
return nil, fmt.Errorf("resource not found")
}
return resource, nil
}
@@ -0,0 +1,73 @@
package service
import (
"context"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func newResourceCatalogForTest(t *testing.T) (interfaces.ResourceCatalog, *gorm.DB) {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.StoredResource{}, &types.ResourceBinding{}, &types.ResourceAccessGrant{}))
return NewResourceCatalog(repository.NewResourceRepository(db)), db
}
func TestResourceCatalogRegisterResolveAndDeduplicate(t *testing.T) {
catalog, _ := newResourceCatalogForTest(t)
ctx := context.Background()
physical := "storage://backend-a/local://7/exports/a.png"
ref, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{Kind: "image", OriginalName: "a.png"})
require.NoError(t, err)
require.Regexp(t, `^resource://[0-9A-Za-z_-]{22}$`, ref)
again, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{})
require.NoError(t, err)
require.Equal(t, ref, again)
resolvedPath, resource, err := catalog.ResolvePath(ctx, ref)
require.NoError(t, err)
require.Equal(t, physical, resolvedPath)
require.Equal(t, uint64(7), resource.TenantID)
require.Equal(t, "backend-a", resource.StorageBackendID)
require.Equal(t, "local", resource.Provider)
}
func TestResourceCatalogBindingAndAccessGrant(t *testing.T) {
catalog, db := newResourceCatalogForTest(t)
ctx := context.Background()
ref, err := catalog.Register(
ctx,
9,
"local://9/exports/report.pdf",
interfaces.ResourceRegistration{OriginalName: "report.pdf"},
)
require.NoError(t, err)
require.NoError(t, catalog.Bind(ctx, ref, "knowledge", "knowledge-1", "source_file"))
token, err := catalog.CreateAccessGrant(ctx, ref, time.Minute)
require.NoError(t, err)
require.Len(t, token, 22)
var storedGrant types.ResourceAccessGrant
require.NoError(t, db.First(&storedGrant).Error)
require.NotEqual(t, token, storedGrant.TokenHash)
require.Len(t, storedGrant.TokenHash, 64)
resource, err := catalog.ResolveAccessGrant(ctx, token)
require.NoError(t, err)
require.Equal(t, uint64(9), resource.TenantID)
}
func TestResourceCatalogRejectsUnsupportedPhysicalPath(t *testing.T) {
catalog, _ := newResourceCatalogForTest(t)
_, err := catalog.Register(context.Background(), 7, "https://example.com/a.png", interfaces.ResourceRegistration{})
require.ErrorContains(t, err, "unsupported provider")
}
+63 -8
View File
@@ -19,12 +19,35 @@ import (
)
type StorageBackendService struct {
repo interfaces.StorageBackendRepository
db *gorm.DB
repo interfaces.StorageBackendRepository
db *gorm.DB
resourceCatalog interfaces.ResourceCatalog
}
func NewStorageBackendService(repo interfaces.StorageBackendRepository, db *gorm.DB) *StorageBackendService {
return &StorageBackendService{repo: repo, db: db}
// NewStorageBackendService creates a storage backend service. The optional
// catalog keeps focused tests compatible while production uses the explicit
// constructor below.
func NewStorageBackendService(
repo interfaces.StorageBackendRepository,
db *gorm.DB,
catalogs ...interfaces.ResourceCatalog,
) *StorageBackendService {
service := &StorageBackendService{repo: repo, db: db}
if len(catalogs) > 0 {
service.resourceCatalog = catalogs[0]
}
return service
}
// NewStorageBackendServiceWithResources is the production DI constructor.
// The variadic constructor above remains convenient for focused tests that do
// not exercise resource registration.
func NewStorageBackendServiceWithResources(
repo interfaces.StorageBackendRepository,
db *gorm.DB,
catalog interfaces.ResourceCatalog,
) *StorageBackendService {
return NewStorageBackendService(repo, db, catalog)
}
func (s *StorageBackendService) Create(ctx context.Context, backend *types.StorageBackend) error {
@@ -76,6 +99,18 @@ func (s *StorageBackendService) Update(ctx context.Context, incoming *types.Stor
return err
}
}
if references == 0 {
if err := s.db.WithContext(ctx).Model(&types.StoredResource{}).
Where(
"tenant_id = ? AND storage_backend_id = ? AND state = ?",
incoming.TenantID,
incoming.ID,
types.ResourceStateActive,
).
Count(&references).Error; err != nil {
return err
}
}
if references > 0 {
return apperrors.NewBadRequestError("a default or bound storage backend cannot be disabled")
}
@@ -123,6 +158,15 @@ func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id
if kbCount > 0 {
return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d knowledge base(s) bound to it", kbCount))
}
var resourceCount int64
if err := tx.Model(&types.StoredResource{}).
Where("tenant_id = ? AND storage_backend_id = ? AND state = ?", tenantID, id, types.ResourceStateActive).
Count(&resourceCount).Error; err != nil {
return err
}
if resourceCount > 0 {
return apperrors.NewBadRequestError(fmt.Sprintf("storage backend still has %d active resource(s)", resourceCount))
}
if backend.LegacyAlias {
return apperrors.NewBadRequestError("legacy storage backend cannot be deleted while old file paths may reference it")
}
@@ -247,12 +291,21 @@ func (s *StorageBackendService) ResolveFileService(ctx context.Context, tenant *
if err != nil {
return nil, provider, err
}
return filesvc.NewBackendScopedFileService(backend.ID, inner), provider, nil
scoped := filesvc.NewBackendScopedFileService(backend.ID, inner)
return filesvc.NewResourceCatalogFileService(scoped, s.resourceCatalog), provider, nil
}
if tenant == nil {
return nil, "", fmt.Errorf("workspace context missing")
}
return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, localBaseDir)
inner, resolvedProvider, err := filesvc.NewFileServiceFromStorageConfig(
provider,
tenant.StorageEngineConfig,
localBaseDir,
)
if err != nil {
return nil, resolvedProvider, err
}
return filesvc.NewResourceCatalogFileService(inner, s.resourceCatalog), resolvedProvider, nil
}
func validateStorageBackendEndpoint(backend *types.StorageBackend) error {
@@ -276,5 +329,7 @@ func validateStorageBackendEndpoint(backend *types.StorageBackend) error {
return nil
}
var _ interfaces.StorageBackendService = (*StorageBackendService)(nil)
var _ interfaces.StorageBackendResolver = (*StorageBackendService)(nil)
var (
_ interfaces.StorageBackendService = (*StorageBackendService)(nil)
_ interfaces.StorageBackendResolver = (*StorageBackendService)(nil)
)
+36 -8
View File
@@ -231,6 +231,8 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(repository.NewWebSearchProviderRepository))
must(container.Provide(repository.NewVectorStoreRepository))
must(container.Provide(repository.NewStorageBackendRepository))
must(container.Provide(repository.NewResourceRepository))
must(container.Provide(service.NewResourceCatalog))
// TenantStoreOwnership adapter used by the retriever factory functions
// to verify that a resolved VectorStore belongs to the caller's tenant.
must(container.Provide(retriever.NewVectorStoreRepoOwnership))
@@ -247,7 +249,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
return sr, nil
}))
must(container.Provide(service.NewVectorStoreService))
must(container.Provide(service.NewStorageBackendService))
must(container.Provide(service.NewStorageBackendServiceWithResources))
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendService { return s }))
must(container.Provide(func(s *service.StorageBackendService) interfaces.StorageBackendResolver { return s }))
@@ -412,20 +414,38 @@ func BuildContainer(container *dig.Container) *dig.Container {
// disk bytes requires rebuilding the FileService from that tenant's storage
// config. The owning tenant is parsed from the URL's first path segment, which
// correctly handles cross-tenant shared resources (e.g. shared KB images).
func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, storageResolver interfaces.StorageBackendResolver) {
func registerChatLocalImageResolver(
tenantRepo interfaces.TenantRepository,
storageResolver interfaces.StorageBackendResolver,
resourceCatalog interfaces.ResourceCatalog,
) {
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
tenantID := secutils.ParseTenantIDFromStoragePath(storageURL)
ctx := context.Background()
physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL)
if err != nil {
return nil, false
}
tenantID := secutils.ParseTenantIDFromStoragePath(physicalPath)
if resource != nil {
tenantID = resource.TenantID
}
if tenantID == 0 {
return nil, false
}
ctx := context.Background()
tenant, err := tenantRepo.GetTenantByID(ctx, tenantID)
if err != nil || tenant == nil {
return nil, false
}
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
backendID, inner, _ := types.ParseStorageBackendPath(storageURL)
provider := types.ParseProviderScheme(inner)
backendID, inner, scoped := types.ParseStorageBackendPath(physicalPath)
if resource != nil && resource.StorageBackendID != "" {
backendID = resource.StorageBackendID
}
providerPath := physicalPath
if scoped {
providerPath = inner
}
provider := types.ParseProviderScheme(providerPath)
if provider == "" {
provider = "local"
}
@@ -433,7 +453,7 @@ func registerChatLocalImageResolver(tenantRepo interfaces.TenantRepository, stor
if err != nil {
return nil, false
}
rc, err := fileSvc.GetFile(ctx, storageURL)
rc, err := fileSvc.GetFile(ctx, physicalPath)
if err != nil {
return nil, false
}
@@ -860,7 +880,15 @@ func syncSequences(db *gorm.DB) {
// Returns:
// - Configured file service implementation
// - Error if initialization fails
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
func initFileService(cfg *config.Config, catalog interfaces.ResourceCatalog) (interfaces.FileService, error) {
inner, err := initRawFileService(cfg)
if err != nil {
return nil, err
}
return file.NewResourceCatalogFileService(inner, catalog), nil
}
func initRawFileService(_ *config.Config) (interfaces.FileService, error) {
storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE"))
if storageType == "" {
storageType = "local"
+9 -2
View File
@@ -79,7 +79,11 @@ func stripImageXMLTags(s string) string {
// storageSchemeRe matches both legacy provider:// URLs and canonical
// storage://<backend-id>/provider:// URLs.
var storageSchemeRe = regexp.MustCompile(`\b(?:storage://[0-9A-Za-z_-]+/)?(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+`)
var storageSchemeRe = regexp.MustCompile(
`\b(?:resource://[0-9A-Za-z_-]+|` +
`(?:storage://[0-9A-Za-z_-]+/)?` +
`(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+)`,
)
// rewriteStorageURLs replaces all provider:// URLs in content with HTTP URLs
// obtained from fileService.GetFileURL. URLs that are already HTTP or cannot
@@ -128,7 +132,7 @@ func rewriteStorageURLs(ctx context.Context, content string, resolver *imFileSer
// incompleteURLSuffixRe matches a provider:// URL that reaches the end of the
// string — it may continue in the next chunk.
var incompleteURLSuffixRe = regexp.MustCompile(
`\b(?:storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`,
`\b(?:resource|storage|local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]*$`,
)
// findIncompleteStorageURL returns the byte offset of a potentially truncated
@@ -248,6 +252,9 @@ func newIMFileServiceResolver(tenant *types.Tenant, defaultSvc interfaces.FileSe
}
func (r *imFileServiceResolver) resolve(filePath string) interfaces.FileService {
if _, ok := types.ParseResourcePath(filePath); ok {
return r.defaultSvc
}
backendID, _, _ := types.ParseStorageBackendPath(filePath)
provider := types.ParseProviderScheme(filePath)
if provider == "" {
+176
View File
@@ -0,0 +1,176 @@
// Package llmresource shortens stored-resource references while they are in an
// LLM context and restores them before application code consumes the output.
package llmresource
import (
"fmt"
"regexp"
"sort"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
// storedRefRE also recognizes legacy physical references. New writes persist
// resource:// handles, but old chunks and message history can still contain a
// provider URL. Giving both forms the same request-local alias makes rollout
// safe without a blocking full-table content rewrite.
var storedRefRE = regexp.MustCompile(
`resource://[0-9A-Za-z_-]{22}|` +
`(?:storage://[0-9A-Za-z_-]+/)?` +
`(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+`,
)
// Registry assigns low-entropy, request-local aliases to stable resource
// handles. It is safe to reuse across all rounds of one Agent execution.
type Registry struct {
mu sync.RWMutex
refToAlias map[string]string
aliasToRef map[string]string
}
// NewRegistry creates an empty request-local alias registry.
func NewRegistry() *Registry {
return &Registry{
refToAlias: make(map[string]string),
aliasToRef: make(map[string]string),
}
}
// EncodeText replaces stored references with compact, stable aliases.
func (r *Registry) EncodeText(value string) string {
if r == nil || value == "" {
return value
}
return storedRefRE.ReplaceAllStringFunc(value, func(ref string) string {
r.mu.Lock()
defer r.mu.Unlock()
if alias, ok := r.refToAlias[ref]; ok {
return alias
}
alias := fmt.Sprintf("res:%04d", len(r.aliasToRef)+1)
r.refToAlias[ref] = alias
r.aliasToRef[alias] = ref
return alias
})
}
// DecodeText restores every alias currently known to the registry.
func (r *Registry) DecodeText(value string) string {
if r == nil || value == "" {
return value
}
r.mu.RLock()
aliases := make([]string, 0, len(r.aliasToRef))
for alias := range r.aliasToRef {
aliases = append(aliases, alias)
}
sort.SliceStable(aliases, func(i, j int) bool { return len(aliases[i]) > len(aliases[j]) })
decoded := value
for _, alias := range aliases {
decoded = strings.ReplaceAll(decoded, alias, r.aliasToRef[alias])
}
r.mu.RUnlock()
return decoded
}
// EncodeMessages returns a copied message slice with textual references
// compacted. Binary/image content fields are intentionally left untouched.
func (r *Registry) EncodeMessages(messages []chat.Message) []chat.Message {
if r == nil || len(messages) == 0 {
return messages
}
encoded := make([]chat.Message, len(messages))
copy(encoded, messages)
for i := range encoded {
encoded[i].Content = r.EncodeText(encoded[i].Content)
encoded[i].ReasoningContent = r.EncodeText(encoded[i].ReasoningContent)
if len(encoded[i].MultiContent) > 0 {
encoded[i].MultiContent = append([]chat.MessageContentPart(nil), encoded[i].MultiContent...)
for j := range encoded[i].MultiContent {
encoded[i].MultiContent[j].Text = r.EncodeText(encoded[i].MultiContent[j].Text)
}
}
if len(encoded[i].ToolCalls) > 0 {
encoded[i].ToolCalls = append([]chat.ToolCall(nil), encoded[i].ToolCalls...)
for j := range encoded[i].ToolCalls {
encoded[i].ToolCalls[j].Function.Arguments = r.EncodeText(encoded[i].ToolCalls[j].Function.Arguments)
}
}
}
return encoded
}
// DecodeResponse restores references in a non-streaming model response.
func (r *Registry) DecodeResponse(response *types.ChatResponse) {
if r == nil || response == nil {
return
}
response.Content = r.DecodeText(response.Content)
response.ReasoningContent = r.DecodeText(response.ReasoningContent)
r.DecodeToolCalls(response.ToolCalls)
}
// DecodeToolCalls restores aliases in tool-call JSON arguments.
func (r *Registry) DecodeToolCalls(toolCalls []types.LLMToolCall) {
for i := range toolCalls {
toolCalls[i].Function.Arguments = r.DecodeText(toolCalls[i].Function.Arguments)
}
}
func (r *Registry) aliases() []string {
r.mu.RLock()
defer r.mu.RUnlock()
aliases := make([]string, 0, len(r.aliasToRef))
for alias := range r.aliasToRef {
aliases = append(aliases, alias)
}
return aliases
}
// StreamDecoder holds only a suffix that could be the beginning of a known
// alias, allowing aliases split across provider chunks to round-trip exactly.
type StreamDecoder struct {
registry *Registry
pending string
}
// NewStreamDecoder creates an alias decoder for one streaming text channel.
func NewStreamDecoder(registry *Registry) *StreamDecoder {
return &StreamDecoder{registry: registry}
}
// Feed decodes a chunk while retaining suffixes that may complete an alias in
// the next provider chunk.
func (d *StreamDecoder) Feed(chunk string) string {
if d == nil || d.registry == nil {
return chunk
}
combined := d.pending + chunk
d.pending = ""
hold := 0
for _, alias := range d.registry.aliases() {
for n := 1; n < len(alias); n++ {
if n > hold && strings.HasSuffix(combined, alias[:n]) {
hold = n
}
}
}
if hold > 0 {
d.pending = combined[len(combined)-hold:]
combined = combined[:len(combined)-hold]
}
return d.registry.DecodeText(combined)
}
// Flush returns any buffered suffix when the provider stream closes.
func (d *StreamDecoder) Flush() string {
if d == nil {
return ""
}
pending := d.pending
d.pending = ""
return d.registry.DecodeText(pending)
}
+43
View File
@@ -0,0 +1,43 @@
package llmresource
import (
"testing"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/stretchr/testify/require"
)
func TestRegistryRoundTripAndDeduplicate(t *testing.T) {
r := NewRegistry()
ref := "resource://AbCdEfGhIjKlMnOpQrStUv"
encoded := r.EncodeText("![a](" + ref + ") and " + ref)
require.Equal(t, "![a](res:0001) and res:0001", encoded)
require.Equal(t, "![a]("+ref+") and "+ref, r.DecodeText(encoded))
}
func TestRegistryAliasesLegacyPhysicalReferencesDuringRollout(t *testing.T) {
r := NewRegistry()
ref := "storage://c0d93536-702c-4977-aa5e-fe670073c3cb/local://10000/exports/image.png"
encoded := r.EncodeText("![image](" + ref + ")")
require.Equal(t, "![image](res:0001)", encoded)
require.Equal(t, "![image]("+ref+")", r.DecodeText(encoded))
}
func TestRegistryEncodesMessageCopies(t *testing.T) {
r := NewRegistry()
ref := "resource://AbCdEfGhIjKlMnOpQrStUv"
original := []chat.Message{{Role: "tool", Content: ref}}
encoded := r.EncodeMessages(original)
require.Equal(t, ref, original[0].Content)
require.Equal(t, "res:0001", encoded[0].Content)
}
func TestStreamDecoderRestoresSplitAlias(t *testing.T) {
r := NewRegistry()
ref := "resource://AbCdEfGhIjKlMnOpQrStUv"
require.Equal(t, "res:0001", r.EncodeText(ref))
d := NewStreamDecoder(r)
require.Equal(t, "before ", d.Feed("before res:0"))
require.Equal(t, ref+" afte", d.Feed("001 after"))
require.Equal(t, "r", d.Flush())
}
+12 -4
View File
@@ -12,12 +12,13 @@ import (
// resolveImageURLForLLM converts stored image paths to a format that LLM APIs can consume.
// - data: URIs and http(s):// URLs are returned as-is.
// - local:// paths are read from disk and converted to base64 data URIs.
// - resource:// and provider-backed local paths are read through the
// application resolver and converted to base64 data URIs.
func resolveImageURLForLLM(imageURL string) string {
if strings.HasPrefix(imageURL, "data:") || strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
return imageURL
}
if strings.HasPrefix(imageURL, "local://") {
if isApplicationStoredImage(imageURL) {
data := readLocalStorageBytes(imageURL)
if data != nil {
mime := http.DetectContentType(data)
@@ -40,14 +41,21 @@ func resolveImageURLForOllama(imageURL string) []byte {
}
return decoded
}
if strings.HasPrefix(imageURL, "local://") {
if isApplicationStoredImage(imageURL) {
return readLocalStorageBytes(imageURL)
}
return nil
}
func isApplicationStoredImage(imageURL string) bool {
return strings.HasPrefix(imageURL, "resource://") ||
strings.HasPrefix(imageURL, "local://") ||
strings.HasPrefix(imageURL, "storage://")
}
// LocalImageResolver, when set by the application layer at startup, resolves a
// local:// storage URL to its bytes using the owning tenant's storage config.
// resource:// or provider storage URL to bytes using the owning tenant's
// storage config.
// Stored local:// URLs are relative to the storage base dir and do NOT encode
// the tenant's configured PathPrefix, so a plain env-based join would miss the
// prefix. When nil (e.g. in tests), callers fall back to the env-based
+212 -19
View File
@@ -76,6 +76,7 @@ type RouterParams struct {
VectorStoreHandler *handler.VectorStoreHandler
StorageBackendHandler *handler.StorageBackendHandler
StorageBackendResolver interfaces.StorageBackendResolver
ResourceCatalog interfaces.ResourceCatalog
FAQHandler *handler.FAQHandler
TagHandler *handler.TagHandler
CustomAgentHandler *handler.CustomAgentHandler
@@ -158,13 +159,26 @@ func NewRouter(params RouterParams) *gin.Engine {
RegisterIMRoutes(r, params.IMHandler)
// Web embed 公开路由(使用 publish token 鉴权,不走全局 Auth)
RegisterEmbedPublicRoutes(r, params.EmbedChannelHandler, params.EmbedChannelService, params.TenantService, params.RedisClient, params.FileService, params.StorageBackendResolver)
RegisterEmbedPublicRoutes(
r,
params.EmbedChannelHandler,
params.EmbedChannelService,
params.TenantService,
params.RedisClient,
params.FileService,
params.StorageBackendResolver,
params.ResourceCatalog,
)
// Short-lived capability URLs for IM and other clients that cannot attach
// WeKnora authentication headers.
serveResourceGrants(r, params.ResourceCatalog, params.TenantService, params.FileService, params.StorageBackendResolver)
// 认证中间件
r.Use(middleware.Auth(params.TenantService, params.UserService, params.TenantMemberService, params.TenantAPIKeyService, params.Config))
// 文件服务:统一代理本地/MinIO/COS/TOS存储后端(需要认证)
serveFiles(r, params.FileService, params.StorageBackendResolver)
serveFilesWithResources(r, params.FileService, params.StorageBackendResolver, params.ResourceCatalog)
// Presigned file access: no auth required, signature-verified.
servePresignedFiles(r, params.TenantService, params.StorageBackendResolver)
@@ -219,7 +233,14 @@ func NewRouter(params RouterParams) *gin.Engine {
// KB-scoped image proxy: lets tenants render images embedded in
// org-shared / agent-visible KB content, which the tenant-scoped
// /files route cannot serve because it enforces same-tenant paths.
serveKBScopedFiles(v1, rbacGuards, params.TenantService, params.FileService, params.StorageBackendResolver)
serveKBScopedFiles(
v1,
rbacGuards,
params.TenantService,
params.FileService,
params.StorageBackendResolver,
params.ResourceCatalog,
)
RegisterKnowledgeTagRoutes(v1, params.TagHandler, rbacGuards)
RegisterKnowledgeRoutes(v1, params.KnowledgeHandler, rbacGuards)
RegisterFAQRoutes(v1, params.FAQHandler, rbacGuards)
@@ -1295,6 +1316,7 @@ func RegisterEmbedPublicRoutes(
redisClient *redis.Client,
fileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalogs ...interfaces.ResourceCatalog,
) {
if embedHandler == nil || embedService == nil {
return
@@ -1322,7 +1344,7 @@ func RegisterEmbedPublicRoutes(
// Serve images embedded in bot replies (e.g. chart exports). EmbedAuth
// injects the channel's tenant, and the handler enforces that the
// requested path belongs to that tenant.
embed.GET("/files", newFileServeHandler(fileService, storageResolver))
embed.GET("/files", newFileServeHandler(fileService, storageResolver, resourceCatalogs...))
}
}
@@ -1500,7 +1522,8 @@ func serveFrontendStatic(r *gin.Engine) {
return
}
path := c.Request.URL.Path
if strings.HasPrefix(path, "/api/") || strings.HasPrefix(path, "/health") || strings.HasPrefix(path, "/swagger/") {
if strings.HasPrefix(path, "/api/") || strings.HasPrefix(path, "/health") || strings.HasPrefix(path, "/swagger/") ||
strings.HasPrefix(path, "/r/") || path == "/files" {
c.Next()
return
}
@@ -1542,7 +1565,15 @@ type getRouteRegistrar interface {
// same handler backs both the authenticated /files route and the embed route
// (where EmbedAuth injects the channel's tenant). Tenant ownership of the
// requested path is enforced via ValidateStoragePathTenant either way.
func newFileServeHandler(globalFileService interfaces.FileService, storageResolver interfaces.StorageBackendResolver) gin.HandlerFunc {
func newFileServeHandler(
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalogs ...interfaces.ResourceCatalog,
) gin.HandlerFunc {
var resourceCatalog interfaces.ResourceCatalog
if len(resourceCatalogs) > 0 {
resourceCatalog = resourceCatalogs[0]
}
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
@@ -1565,6 +1596,27 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv
return
}
tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"})
return
}
resourceResolved := false
if resourceCatalog != nil {
resolvedPath, resource, err := resourceCatalog.ResolvePath(c.Request.Context(), filePath)
if err != nil {
c.Status(http.StatusNotFound)
return
}
if resource != nil {
if resource.TenantID != tenant.ID {
c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: resource not accessible"})
return
}
filePath = resolvedPath
resourceResolved = true
}
}
backendID, innerPath, scoped := types.ParseStorageBackendPath(filePath)
providerPath := filePath
if scoped {
@@ -1572,16 +1624,21 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv
}
provider := types.ParseProviderScheme(providerPath)
tenant, _ := c.Request.Context().Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"})
return
}
if err := secutils.ValidateStoragePathTenant(filePath, tenant.ID); err != nil {
logger.Warnf(context.Background(), "[Router] /files denied cross-tenant or invalid path: tenant_id=%d file_path=%q err=%v", tenant.ID, filePath, err)
c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"})
return
// A registered resource's tenant is authoritative. Physical provider
// paths remain an internal locator and are not required to encode access
// control metadata (some cloud layouts contain other numeric segments).
if !resourceResolved {
if err := secutils.ValidateStoragePathTenant(filePath, tenant.ID); err != nil {
logger.Warnf(
context.Background(),
"[Router] /files denied cross-tenant or invalid path: tenant_id=%d file_path=%q err=%v",
tenant.ID,
filePath,
err,
)
c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"})
return
}
}
var (
@@ -1636,15 +1693,115 @@ func newFileServeHandler(globalFileService interfaces.FileService, storageResolv
}
func serveFiles(r getRouteRegistrar, globalFileService interfaces.FileService, resolvers ...interfaces.StorageBackendResolver) {
logger.Infof(context.Background(), "[Router] Serving files from /files")
var storageResolver interfaces.StorageBackendResolver
if len(resolvers) > 0 {
storageResolver = resolvers[0]
}
serveFilesWithResources(r, globalFileService, storageResolver, nil)
}
func serveFilesWithResources(
r getRouteRegistrar,
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalog interfaces.ResourceCatalog,
) {
logger.Infof(context.Background(), "[Router] Serving files from /files")
// /files sits outside the /api/v1 APIKeyGate; storage paths cannot be
// attributed to a key's KB allow-list, so API keys are denied outright
// (embed routes use their own /embed/.../files handler).
r.GET("/files", middleware.DenyAPIKeyPrincipal(), newFileServeHandler(globalFileService, storageResolver))
r.GET(
"/files",
middleware.DenyAPIKeyPrincipal(),
newFileServeHandler(globalFileService, storageResolver, resourceCatalog),
)
}
// serveResourceGrants exposes short, revocable capability URLs for clients
// such as IM platforms that cannot attach a WeKnora bearer/embed token.
func serveResourceGrants(
r *gin.Engine,
resourceCatalog interfaces.ResourceCatalog,
tenantService interfaces.TenantService,
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
) {
if resourceCatalog == nil || tenantService == nil {
return
}
handler := func(c *gin.Context) {
ctx := c.Request.Context()
resource, err := resourceCatalog.ResolveAccessGrant(ctx, c.Param("token"))
if err != nil || resource == nil {
c.Status(http.StatusNotFound)
return
}
tenant, err := tenantService.GetTenantByID(ctx, resource.TenantID)
if err != nil || tenant == nil {
c.Status(http.StatusNotFound)
return
}
providerPath := resource.PhysicalPath
if _, inner, ok := types.ParseStorageBackendPath(providerPath); ok {
providerPath = inner
}
provider := types.ParseProviderScheme(providerPath)
var fileSvc interfaces.FileService
if storageResolver != nil {
fileSvc, _, err = storageResolver.ResolveFileService(
ctx,
tenant,
resource.StorageBackendID,
provider,
localStorageBaseDir(),
)
} else {
fileSvc = globalFileService
}
if err != nil || fileSvc == nil {
logger.Warnf(ctx, "[Router] resource grant storage resolution failed: resource_id=%s err=%v", resource.ID, err)
c.Status(http.StatusNotFound)
return
}
reader, err := fileSvc.GetFile(ctx, resource.PhysicalPath)
if err != nil {
c.Status(http.StatusNotFound)
return
}
defer func() { _ = reader.Close() }()
fileName := resource.OriginalName
if fileName == "" {
fileName = resource.PhysicalPath
}
contentType, inline := secutils.SafeContentTypeByFilename(fileName)
if resource.MimeType != "" && inline {
contentType = resource.MimeType
}
c.Header("Content-Type", contentType)
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Cache-Control", "private, max-age=300")
if !inline {
c.Header("Content-Disposition", "attachment")
}
c.Status(http.StatusOK)
if c.Request.Method != http.MethodHead {
if _, err := io.Copy(c.Writer, reader); err != nil {
logger.Warnf(ctx, "[Router] resource grant write failed: resource_id=%s err=%v", resource.ID, err)
}
}
}
r.GET("/r/:token", handler)
r.HEAD("/r/:token", handler)
}
func localStorageBaseDir() string {
baseDir := strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
if baseDir == "" {
baseDir = "/data/files"
}
return baseDir
}
// serveKBScopedFiles registers the KB-scoped file proxy used to render images
@@ -1665,6 +1822,7 @@ func serveKBScopedFiles(
tenantService interfaces.TenantService,
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalogs ...interfaces.ResourceCatalog,
) {
logger.Infof(context.Background(), "[Router] Serving KB-scoped files from /knowledge-bases/:id/files")
// API keys are denied outright (as on /files): a signed URL's tenant/KB
@@ -1674,7 +1832,12 @@ func serveKBScopedFiles(
middleware.DenyAPIKeyPrincipal(),
g.Viewer(),
g.KBAccessRead("id"),
newKBScopedFileServeHandler(tenantService, globalFileService, storageResolver),
newKBScopedFileServeHandlerWithResources(
tenantService,
globalFileService,
storageResolver,
firstResourceCatalog(resourceCatalogs),
),
)
}
@@ -1693,6 +1856,22 @@ func newKBScopedFileServeHandler(
if len(resolvers) > 0 {
storageResolver = resolvers[0]
}
return newKBScopedFileServeHandlerWithResources(tenantService, globalFileService, storageResolver, nil)
}
func firstResourceCatalog(catalogs []interfaces.ResourceCatalog) interfaces.ResourceCatalog {
if len(catalogs) == 0 {
return nil
}
return catalogs[0]
}
func newKBScopedFileServeHandlerWithResources(
tenantService interfaces.TenantService,
globalFileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
resourceCatalog interfaces.ResourceCatalog,
) gin.HandlerFunc {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
@@ -1721,6 +1900,20 @@ func newKBScopedFileServeHandler(
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"})
return
}
if resourceCatalog != nil {
resolvedPath, resource, err := resourceCatalog.ResolvePath(ctx, filePath)
if err != nil {
c.Status(http.StatusNotFound)
return
}
if resource != nil {
if resource.TenantID != ownerTenantID {
c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: resource not accessible"})
return
}
filePath = resolvedPath
}
}
if err := secutils.ValidateKBScopedStoragePath(filePath, ownerTenantID); err != nil {
logger.Warnf(ctx, "[Router] /knowledge-bases/:id/files denied path not allowed for KB proxy: owner_tenant_id=%d file_path=%q err=%v",
+122
View File
@@ -9,6 +9,7 @@ import (
"net/url"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
@@ -22,6 +23,46 @@ type stubFileService struct {
getFile func(ctx context.Context, filePath string) (io.ReadCloser, error)
}
type stubResourceCatalog struct {
resource *types.StoredResource
}
func (s *stubResourceCatalog) Register(
context.Context,
uint64,
string,
interfaces.ResourceRegistration,
) (string, error) {
panic("unexpected Register")
}
func (s *stubResourceCatalog) Resolve(context.Context, string) (*types.StoredResource, error) {
return s.resource, nil
}
func (s *stubResourceCatalog) ResolvePath(_ context.Context, value string) (string, *types.StoredResource, error) {
if _, ok := types.ParseResourcePath(value); ok && s.resource != nil {
return s.resource.PhysicalPath, s.resource, nil
}
return value, nil, nil
}
func (s *stubResourceCatalog) Bind(context.Context, string, string, string, string) error {
panic("unexpected Bind")
}
func (s *stubResourceCatalog) MarkDeleted(context.Context, string) error {
panic("unexpected MarkDeleted")
}
func (s *stubResourceCatalog) CreateAccessGrant(context.Context, string, time.Duration) (string, error) {
panic("unexpected CreateAccessGrant")
}
func (s *stubResourceCatalog) ResolveAccessGrant(context.Context, string) (*types.StoredResource, error) {
return s.resource, nil
}
func (s *stubFileService) CheckConnectivity(ctx context.Context) error {
return nil
}
@@ -84,6 +125,87 @@ func TestServeFilesFallsBackToGlobalFileService(t *testing.T) {
}
}
func TestServeFilesResolvesShortResourceReference(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("STORAGE_TYPE", "local")
const ref = "resource://AbCdEfGhIjKlMnOpQrStUv"
const physical = "local://42/exports/a.png"
engine := gin.New()
var requestedPath string
serveFilesWithResources(engine, &stubFileService{getFile: func(_ context.Context, path string) (io.ReadCloser, error) {
requestedPath = path
return io.NopCloser(strings.NewReader("image")), nil
}}, nil, &stubResourceCatalog{resource: &types.StoredResource{TenantID: 42, PhysicalPath: physical}})
req := httptest.NewRequest(http.MethodGet, "/files?file_path="+url.QueryEscape(ref), nil)
req = req.WithContext(context.WithValue(req.Context(), types.TenantInfoContextKey, &types.Tenant{ID: 42}))
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, req)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if requestedPath != physical {
t.Fatalf("requested path = %q, want %q", requestedPath, physical)
}
}
func TestServeFilesRejectsCrossTenantResourceReference(t *testing.T) {
gin.SetMode(gin.TestMode)
const ref = "resource://AbCdEfGhIjKlMnOpQrStUv"
engine := gin.New()
serveFilesWithResources(engine, &stubFileService{getFile: func(context.Context, string) (io.ReadCloser, error) {
t.Fatal("GetFile should not be called")
return nil, nil
}}, nil, &stubResourceCatalog{resource: &types.StoredResource{TenantID: 7, PhysicalPath: "local://7/exports/a.png"}})
req := httptest.NewRequest(http.MethodGet, "/files?file_path="+url.QueryEscape(ref), nil)
req = req.WithContext(context.WithValue(req.Context(), types.TenantInfoContextKey, &types.Tenant{ID: 42}))
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, req)
if recorder.Code != http.StatusForbidden {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusForbidden)
}
}
func TestResourceGrantServesShortPublicURL(t *testing.T) {
gin.SetMode(gin.TestMode)
physical := "local://42/exports/a.png"
engine := gin.New()
serveResourceGrants(
engine,
&stubResourceCatalog{resource: &types.StoredResource{
ID: "resource-1",
TenantID: 42,
PhysicalPath: physical,
OriginalName: "a.png",
MimeType: "image/png",
}},
&stubTenantService{get: func(_ context.Context, id uint64) (*types.Tenant, error) {
return &types.Tenant{ID: id}, nil
}},
&stubFileService{getFile: func(_ context.Context, path string) (io.ReadCloser, error) {
if path != physical {
t.Fatalf("path = %q, want %q", path, physical)
}
return io.NopCloser(strings.NewReader("image")), nil
}},
nil,
)
req := httptest.NewRequest(http.MethodGet, "/r/GrantTokenAbCdEfGhIjKlM", nil)
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, req)
if recorder.Code != http.StatusOK || recorder.Body.String() != "image" {
t.Fatalf("status=%d body=%q", recorder.Code, recorder.Body.String())
}
if got := recorder.Header().Get("X-Content-Type-Options"); got != "nosniff" {
t.Fatalf("X-Content-Type-Options = %q", got)
}
}
func TestServeFilesDoesNotFallbackWhenProviderDoesNotMatchGlobalStorage(t *testing.T) {
gin.SetMode(gin.TestMode)
t.Setenv("STORAGE_TYPE", "minio")
+30
View File
@@ -0,0 +1,30 @@
package router
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestFrontendStaticDoesNotInterceptResourceGrant(t *testing.T) {
gin.SetMode(gin.TestMode)
webDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(webDir, "index.html"), []byte("spa"), 0o600))
t.Setenv("WEKNORA_WEB_DIR", webDir)
r := gin.New()
serveFrontendStatic(r)
r.GET("/r/:token", func(c *gin.Context) { c.String(http.StatusOK, "resource") })
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/r/token", nil)
r.ServeHTTP(recorder, request)
require.Equal(t, http.StatusOK, recorder.Code)
require.Equal(t, "resource", recorder.Body.String())
}
+39 -6
View File
@@ -2,18 +2,51 @@ package interfaces
import (
"context"
"time"
"github.com/Tencent/WeKnora/internal/types"
)
// ResourceCleaner defines the resource cleaner interface
// ResourceCleaner owns process-lifetime cleanup callbacks (database clients,
// worker pools, etc.). It is unrelated to persisted StoredResource records.
type ResourceCleaner interface {
// Register registers a resource cleanup function
Register(cleanup types.CleanupFunc)
// RegisterWithName registers a resource cleanup function with a name
RegisterWithName(name string, cleanup types.CleanupFunc)
// Cleanup executes all resource cleanup functions
Cleanup(ctx context.Context) []error
}
// ResourceRepository persists stable resource identities, ownership bindings,
// and short-lived access grants.
type ResourceRepository interface {
Create(ctx context.Context, resource *types.StoredResource) error
GetByID(ctx context.Context, id string) (*types.StoredResource, error)
GetByHandle(ctx context.Context, handle string) (*types.StoredResource, error)
GetByTenantLocation(ctx context.Context, tenantID uint64, locationHash string) (*types.StoredResource, error)
MarkDeleted(ctx context.Context, id string) error
CreateBinding(ctx context.Context, binding *types.ResourceBinding) error
CreateGrant(ctx context.Context, grant *types.ResourceAccessGrant) error
GetValidGrant(ctx context.Context, tokenHash string, now time.Time) (*types.ResourceAccessGrant, error)
DeleteExpiredGrants(ctx context.Context, before time.Time) error
}
// ResourceRegistration describes one physical object at registration time.
type ResourceRegistration struct {
Kind string
MimeType string
OriginalName string
Size int64
ContentHash string
Temporary bool
}
// ResourceCatalog maps public resource references to internal storage
// locations and manages their access capabilities.
type ResourceCatalog interface {
Register(ctx context.Context, tenantID uint64, physicalPath string, meta ResourceRegistration) (string, error)
Resolve(ctx context.Context, reference string) (*types.StoredResource, error)
ResolvePath(ctx context.Context, value string) (string, *types.StoredResource, error)
Bind(ctx context.Context, reference, ownerType, ownerID, relation string) error
MarkDeleted(ctx context.Context, reference string) error
CreateAccessGrant(ctx context.Context, reference string, ttl time.Duration) (string, error)
ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error)
}
+1 -1
View File
@@ -384,7 +384,7 @@ func ParseProviderScheme(filePath string) string {
if _, inner, ok := ParseStorageBackendPath(filePath); ok {
filePath = inner
}
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs"} {
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss", "ks3", "obs", "dummy"} {
if strings.HasPrefix(filePath, provider+"://") {
return provider
}
+131
View File
@@ -0,0 +1,131 @@
package types
import (
"strings"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
)
// Stable resource reference and lifecycle constants.
const (
ResourceScheme = "resource://"
ResourceHandleLength = 22
ResourceStateActive = "active"
ResourceStateDeleted = "deleted"
ResourceLifecyclePersistent = "persistent"
ResourceLifecycleTemporary = "temporary"
)
// StoredResource is the stable application identity of one stored object. PhysicalPath
// is deliberately internal: API responses, persisted rich text and LLM prompts
// use resource://<handle> instead.
type StoredResource struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
Handle string `json:"handle" gorm:"type:varchar(22);not null;uniqueIndex"`
TenantID uint64 `json:"tenant_id" gorm:"not null;index"`
StorageBackendID string `json:"storage_backend_id,omitempty" gorm:"type:varchar(36);index"`
Provider string `json:"provider" gorm:"type:varchar(32);not null"`
PhysicalPath string `json:"-" gorm:"type:text;not null"`
LocationHash string `json:"-" gorm:"type:varchar(64);not null"`
Kind string `json:"kind" gorm:"type:varchar(32);not null;default:'file'"`
MimeType string `json:"mime_type,omitempty" gorm:"type:varchar(255);not null;default:''"`
OriginalName string `json:"original_name,omitempty" gorm:"type:varchar(1024);not null;default:''"`
Size int64 `json:"size" gorm:"not null;default:0"`
ContentHash string `json:"content_hash,omitempty" gorm:"type:varchar(64);not null;default:''"`
Lifecycle string `json:"lifecycle" gorm:"type:varchar(16);not null;default:'persistent'"`
ExpiresAt *time.Time `json:"expires_at,omitempty"`
State string `json:"state" gorm:"type:varchar(16);not null;default:'active'"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
}
// TableName returns the resource registry table name.
func (StoredResource) TableName() string { return "resources" }
// BeforeCreate fills resource defaults before persistence.
func (r *StoredResource) BeforeCreate(_ *gorm.DB) error {
if r.ID == "" {
r.ID = uuid.NewString()
}
if r.Kind == "" {
r.Kind = "file"
}
if r.Lifecycle == "" {
r.Lifecycle = ResourceLifecyclePersistent
}
if r.State == "" {
r.State = ResourceStateActive
}
return nil
}
// ResourceBinding connects a resource to a domain object that owns or uses it.
type ResourceBinding struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
ResourceID string `json:"resource_id" gorm:"type:varchar(36);not null;index"`
TenantID uint64 `json:"tenant_id" gorm:"not null;index"`
OwnerType string `json:"owner_type" gorm:"type:varchar(32);not null"`
OwnerID string `json:"owner_id" gorm:"type:varchar(64);not null"`
Relation string `json:"relation" gorm:"type:varchar(32);not null;default:'attachment'"`
CreatedAt time.Time `json:"created_at"`
}
// TableName returns the resource binding table name.
func (ResourceBinding) TableName() string { return "resource_bindings" }
// BeforeCreate assigns an ID before persistence.
func (b *ResourceBinding) BeforeCreate(_ *gorm.DB) error {
if b.ID == "" {
b.ID = uuid.NewString()
}
return nil
}
// ResourceAccessGrant is a revocable, expiring read capability.
type ResourceAccessGrant struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
TokenHash string `json:"-" gorm:"type:varchar(64);not null;uniqueIndex"`
ResourceID string `json:"resource_id" gorm:"type:varchar(36);not null;index"`
AccessScope string `json:"access_scope" gorm:"type:varchar(16);not null;default:'read'"`
ExpiresAt time.Time `json:"expires_at" gorm:"not null;index"`
RevokedAt *time.Time `json:"revoked_at,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
// TableName returns the resource access grant table name.
func (ResourceAccessGrant) TableName() string { return "resource_access_grants" }
// BeforeCreate assigns an ID before persistence.
func (g *ResourceAccessGrant) BeforeCreate(_ *gorm.DB) error {
if g.ID == "" {
g.ID = uuid.NewString()
}
return nil
}
// BuildResourcePath creates a stable application resource reference.
func BuildResourcePath(handle string) string {
return ResourceScheme + strings.TrimSpace(handle)
}
// ParseResourcePath validates and extracts a canonical resource handle.
func ParseResourcePath(value string) (string, bool) {
value = strings.TrimSpace(value)
if !strings.HasPrefix(value, ResourceScheme) {
return "", false
}
handle := strings.TrimPrefix(value, ResourceScheme)
if len(handle) != ResourceHandleLength {
return "", false
}
for _, char := range handle {
if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') &&
(char < '0' || char > '9') && char != '_' && char != '-' {
return "", false
}
}
return handle, true
}
+9 -2
View File
@@ -164,8 +164,12 @@ func IsValidURL(url string) bool {
return false
}
// 检查协议, 只允许 http, https, local, minio, cos, tos, oss 协议
allowedProtocols := []string{"http://", "https://", "local://", "minio://", "cos://", "tos://", "oss://"}
// Internal resource references are resolved through authenticated file
// proxies; provider schemes remain supported for legacy stored content.
allowedProtocols := []string{
"http://", "https://", "resource://", "storage://", "local://", "minio://",
"cos://", "tos://", "s3://", "oss://", "ks3://", "obs://",
}
isAllowed := false
for _, protocol := range allowedProtocols {
if strings.HasPrefix(strings.ToLower(url), protocol) {
@@ -486,6 +490,9 @@ func IsValidImageURL(url string) bool {
if !IsValidURL(url) {
return false
}
if strings.HasPrefix(strings.ToLower(url), "resource://") {
return true
}
// 检查是否为图片文件
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"}
+3
View File
@@ -22,5 +22,8 @@ DROP TABLE IF EXISTS sessions;
DROP TABLE IF EXISTS knowledges;
DROP TABLE IF EXISTS knowledge_bases;
DROP TABLE IF EXISTS storage_backends;
DROP TABLE IF EXISTS resource_access_grants;
DROP TABLE IF EXISTS resource_bindings;
DROP TABLE IF EXISTS resources;
DROP TABLE IF EXISTS models;
DROP TABLE IF EXISTS tenants;
+53
View File
@@ -827,6 +827,59 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_backends_legacy_alias
ON storage_backends(tenant_id, provider) WHERE deleted_at IS NULL AND legacy_alias = 1;
CREATE INDEX IF NOT EXISTS idx_storage_backends_tenant ON storage_backends(tenant_id);
CREATE TABLE IF NOT EXISTS resources (
id VARCHAR(36) PRIMARY KEY,
handle VARCHAR(22) NOT NULL UNIQUE,
tenant_id INTEGER NOT NULL,
storage_backend_id VARCHAR(36),
provider VARCHAR(32) NOT NULL,
physical_path TEXT NOT NULL,
location_hash VARCHAR(64) NOT NULL,
kind VARCHAR(32) NOT NULL DEFAULT 'file',
mime_type VARCHAR(255) NOT NULL DEFAULT '',
original_name VARCHAR(1024) NOT NULL DEFAULT '',
size INTEGER NOT NULL DEFAULT 0,
content_hash VARCHAR(64) NOT NULL DEFAULT '',
lifecycle VARCHAR(16) NOT NULL DEFAULT 'persistent',
expires_at DATETIME,
state VARCHAR(16) NOT NULL DEFAULT 'active',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
deleted_at DATETIME
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_resources_tenant_location
ON resources(tenant_id, location_hash) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_resources_tenant ON resources(tenant_id);
CREATE INDEX IF NOT EXISTS idx_resources_backend ON resources(storage_backend_id);
CREATE TABLE IF NOT EXISTS resource_bindings (
id VARCHAR(36) PRIMARY KEY,
resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE,
tenant_id INTEGER NOT NULL,
owner_type VARCHAR(32) NOT NULL,
owner_id VARCHAR(64) NOT NULL,
relation VARCHAR(32) NOT NULL DEFAULT 'attachment',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_resource_bindings_unique
ON resource_bindings(resource_id, owner_type, owner_id, relation);
CREATE INDEX IF NOT EXISTS idx_resource_bindings_owner
ON resource_bindings(tenant_id, owner_type, owner_id);
CREATE TABLE IF NOT EXISTS resource_access_grants (
id VARCHAR(36) PRIMARY KEY,
token_hash VARCHAR(64) NOT NULL UNIQUE,
resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE,
access_scope VARCHAR(16) NOT NULL DEFAULT 'read',
expires_at DATETIME NOT NULL,
revoked_at DATETIME,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_resource_access_grants_resource
ON resource_access_grants(resource_id);
CREATE INDEX IF NOT EXISTS idx_resource_access_grants_expires
ON resource_access_grants(expires_at);
CREATE TABLE IF NOT EXISTS tenant_api_keys (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id INTEGER NOT NULL,
@@ -0,0 +1,3 @@
DROP TABLE IF EXISTS resource_access_grants;
DROP TABLE IF EXISTS resource_bindings;
DROP TABLE IF EXISTS resources;
@@ -0,0 +1,55 @@
CREATE TABLE IF NOT EXISTS resources (
id VARCHAR(36) NOT NULL PRIMARY KEY,
handle VARCHAR(22) NOT NULL UNIQUE,
tenant_id BIGINT NOT NULL,
storage_backend_id VARCHAR(36),
provider VARCHAR(32) NOT NULL,
physical_path TEXT NOT NULL,
location_hash VARCHAR(64) NOT NULL,
kind VARCHAR(32) NOT NULL DEFAULT 'file',
mime_type VARCHAR(255) NOT NULL DEFAULT '',
original_name VARCHAR(1024) NOT NULL DEFAULT '',
size BIGINT NOT NULL DEFAULT 0,
content_hash VARCHAR(64) NOT NULL DEFAULT '',
lifecycle VARCHAR(16) NOT NULL DEFAULT 'persistent',
expires_at TIMESTAMP NULL,
state VARCHAR(16) NOT NULL DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_resources_tenant_location
ON resources(tenant_id, location_hash) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_resources_tenant ON resources(tenant_id);
CREATE INDEX IF NOT EXISTS idx_resources_backend ON resources(storage_backend_id);
CREATE TABLE IF NOT EXISTS resource_bindings (
id VARCHAR(36) NOT NULL PRIMARY KEY,
resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE,
tenant_id BIGINT NOT NULL,
owner_type VARCHAR(32) NOT NULL,
owner_id VARCHAR(64) NOT NULL,
relation VARCHAR(32) NOT NULL DEFAULT 'attachment',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_resource_bindings_unique
ON resource_bindings(resource_id, owner_type, owner_id, relation);
CREATE INDEX IF NOT EXISTS idx_resource_bindings_owner
ON resource_bindings(tenant_id, owner_type, owner_id);
CREATE TABLE IF NOT EXISTS resource_access_grants (
id VARCHAR(36) NOT NULL PRIMARY KEY,
token_hash VARCHAR(64) NOT NULL UNIQUE,
resource_id VARCHAR(36) NOT NULL REFERENCES resources(id) ON DELETE CASCADE,
access_scope VARCHAR(16) NOT NULL DEFAULT 'read',
expires_at TIMESTAMP NOT NULL,
revoked_at TIMESTAMP NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_resource_access_grants_resource
ON resource_access_grants(resource_id);
CREATE INDEX IF NOT EXISTS idx_resource_access_grants_expires
ON resource_access_grants(expires_at);