feat(embed): add backend embed channels with auth and rate limiting (#1657)

Introduce embed_channels storage, publish-token and session-token auth,
origin allowlist enforcement, shared Redis rate limiter, public embed API
routes, webhooks with SSRF validation, and session listing by embed source.
This commit is contained in:
wizardchen
2026-06-12 22:40:18 +08:00
committed by lyingbug
parent a0786b6fe7
commit ed9802ff2b
30 changed files with 4070 additions and 216 deletions
+1 -1
View File
@@ -24,7 +24,7 @@ import (
// Local in-memory only — fine for typical deployments since both
// endpoints handle low absolute volumes; if/when WeKnora horizontally
// scales the auth surface, swap to the Redis-backed limiter in
// internal/im/ratelimit.go (kept lowercase / package-private today).
// internal/ratelimit (shared with IM + embed surfaces).
// publicAuthRateLimitWindow is the rolling window length per IP.
const publicAuthRateLimitWindow = 60 * time.Second
+247
View File
@@ -0,0 +1,247 @@
package middleware
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/ratelimit"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
"github.com/redis/go-redis/v9"
)
const (
embedRateLimitKeyPrefix = "embed:ratelimit:"
embedDailyRateLimitKeyPrefix = "embed:ratelimit:day:"
// embedGlobalMinuteFactor derives a channel-wide per-minute cap from the
// per-IP cap. The publish token is publicly visible, so a single attacker
// can rotate IPs to defeat the per-IP limit; this bounds aggregate burst.
embedGlobalMinuteFactor = 20
// embedGlobalMinuteFloor keeps the global per-minute cap usable even when
// the per-IP cap is tiny.
embedGlobalMinuteFloor = 120
)
// EmbedChannelContextKey stores the authenticated embed channel on the request context.
const EmbedChannelContextKey types.ContextKey = "EmbedChannel"
var (
embedLimiterOnce sync.Once
embedLimiter *ratelimit.Limiter
embedDailyLimiterOnce sync.Once
embedDailyLimiter *ratelimit.Limiter
)
func embedRateLimiter(redisClient *redis.Client) *ratelimit.Limiter {
embedLimiterOnce.Do(func() {
embedLimiter = ratelimit.New(redisClient, embedRateLimitKeyPrefix, time.Minute, "")
// Local-fallback eviction; Redis keys expire via PEXPIRE in the Lua script.
stopCh := make(chan struct{})
go embedLimiter.StartCleanup(stopCh)
})
return embedLimiter
}
func embedDailyRateLimiter(redisClient *redis.Client) *ratelimit.Limiter {
embedDailyLimiterOnce.Do(func() {
embedDailyLimiter = ratelimit.New(redisClient, embedDailyRateLimitKeyPrefix, 24*time.Hour, "")
stopCh := make(chan struct{})
go embedDailyLimiter.StartCleanup(stopCh)
})
return embedDailyLimiter
}
// embedGlobalPerMinute returns the channel-wide per-minute budget derived from
// the per-IP budget.
func embedGlobalPerMinute(perIP int) int {
budget := perIP * embedGlobalMinuteFactor
if budget < embedGlobalMinuteFloor {
budget = embedGlobalMinuteFloor
}
return budget
}
// EmbedAuth validates publish tokens and injects a scoped tenant context for embed routes.
func EmbedAuth(
svc interfaces.EmbedChannelService,
tenantSvc interfaces.TenantService,
redisClient *redis.Client,
) gin.HandlerFunc {
limiter := embedRateLimiter(redisClient)
dailyLimiter := embedDailyRateLimiter(redisClient)
return func(c *gin.Context) {
channelID := strings.TrimSpace(c.Param("channel_id"))
if channelID == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "channel_id is required"})
c.Abort()
return
}
token := extractEmbedToken(c)
if token == "" {
c.JSON(http.StatusUnauthorized, gin.H{"error": "embed publish token is required"})
c.Abort()
return
}
var ch *types.EmbedChannel
var err error
if service.IsEmbedSessionToken(token) {
resolvedID, resolveErr := svc.ResolveSessionToken(c.Request.Context(), token)
if resolveErr != nil || resolvedID != channelID {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid embed channel or token"})
c.Abort()
return
}
ch, err = svc.LookupEnabledChannel(c.Request.Context(), channelID)
} else {
ch, err = svc.LookupForEmbed(c.Request.Context(), channelID, token)
}
if err != nil {
if errors.Is(err, service.ErrEmbedChannelDisabled) {
c.JSON(http.StatusForbidden, gin.H{"error": "embed channel is disabled"})
c.Abort()
return
}
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid embed channel or token"})
c.Abort()
return
}
origin := requestOrigin(c)
if !originAllowed(origin, ch.AllowedOriginsList()) {
logger.Warnf(c.Request.Context(), "[embed_auth] origin %q not allowed for channel %s", origin, channelID)
c.JSON(http.StatusForbidden, gin.H{"error": "origin not allowed"})
c.Abort()
return
}
// Per-IP per-minute cap.
rateKey := fmt.Sprintf("%s:%s", channelID, c.ClientIP())
if !limiter.Allow(c.Request.Context(), rateKey, ch.RateLimitPerMinute) {
c.JSON(http.StatusTooManyRequests, gin.H{"error": "rate limit exceeded"})
c.Abort()
return
}
// Channel-wide per-minute cap (bounds burst across rotating IPs since
// the publish token is publicly visible).
if !limiter.Allow(c.Request.Context(), channelID+":__global", embedGlobalPerMinute(ch.RateLimitPerMinute)) {
c.JSON(http.StatusTooManyRequests, gin.H{"error": "rate limit exceeded"})
c.Abort()
return
}
// Channel-wide daily total cap (bounds sustained abuse).
if !dailyLimiter.Allow(c.Request.Context(), channelID, ch.RateLimitPerDay) {
c.JSON(http.StatusTooManyRequests, gin.H{"error": "daily request limit exceeded"})
c.Abort()
return
}
tenant, err := tenantSvc.GetTenantByID(c.Request.Context(), ch.TenantID)
if err != nil || tenant == nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "tenant unavailable"})
c.Abort()
return
}
user := &types.User{
ID: fmt.Sprintf("embed-%s", channelID),
Username: fmt.Sprintf("embed-%s", channelID),
Email: fmt.Sprintf("embed-%s@embed.local", channelID),
TenantID: ch.TenantID,
IsActive: true,
}
c.Set(types.TenantIDContextKey.String(), ch.TenantID)
c.Set(types.TenantInfoContextKey.String(), tenant)
c.Set(types.UserContextKey.String(), user)
c.Set(types.UserIDContextKey.String(), user.ID)
c.Set(types.TenantRoleContextKey.String(), types.TenantRoleViewer)
c.Set(types.SystemAdminContextKey.String(), false)
c.Set(string(EmbedChannelContextKey), ch)
ctx := c.Request.Context()
ctx = context.WithValue(ctx, types.TenantIDContextKey, ch.TenantID)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
ctx = context.WithValue(ctx, types.UserContextKey, user)
ctx = context.WithValue(ctx, types.UserIDContextKey, user.ID)
ctx = context.WithValue(ctx, types.TenantRoleContextKey, types.TenantRoleViewer)
ctx = context.WithValue(ctx, types.SystemAdminContextKey, false)
ctx = context.WithValue(ctx, EmbedChannelContextKey, ch)
c.Request = c.Request.WithContext(ctx)
c.Next()
}
}
func extractEmbedToken(c *gin.Context) string {
// Only accept the token via the Authorization header. A query-string token
// would be captured by proxy/access logs and browser history; the embed
// client always sends "Authorization: Embed <token>".
auth := c.GetHeader("Authorization")
if strings.HasPrefix(auth, "Embed ") {
return strings.TrimSpace(strings.TrimPrefix(auth, "Embed "))
}
return ""
}
func requestOrigin(c *gin.Context) string {
if o := strings.TrimSpace(c.GetHeader("Origin")); o != "" {
return o
}
ref := strings.TrimSpace(c.GetHeader("Referer"))
if ref == "" {
return ""
}
u, err := url.Parse(ref)
if err != nil {
return ""
}
if u.Scheme == "" || u.Host == "" {
return ""
}
return u.Scheme + "://" + u.Host
}
func originAllowed(origin string, allowed []string) bool {
// Empty allowlist rejects all origins. Management create/update requires at
// least one origin; legacy rows with [] must be fixed before going live.
if len(allowed) == 0 {
return false
}
if origin == "" {
return false
}
for _, pattern := range allowed {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
continue
}
if pattern == "*" || strings.EqualFold(pattern, origin) {
return true
}
if strings.HasPrefix(pattern, "*.") {
suffix := strings.TrimPrefix(pattern, "*")
if strings.HasSuffix(origin, suffix) {
return true
}
}
}
return false
}
// EmbedChannelFromContext returns the authenticated embed channel, if any.
func EmbedChannelFromContext(ctx context.Context) (*types.EmbedChannel, bool) {
ch, ok := ctx.Value(EmbedChannelContextKey).(*types.EmbedChannel)
return ch, ok && ch != nil
}
+408
View File
@@ -0,0 +1,408 @@
package middleware
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
)
type fakeEmbedChannelService struct {
channels map[string]*types.EmbedChannel
sessions map[string]string
}
func (f *fakeEmbedChannelService) Create(
ctx context.Context, tenantID uint64, agentID string, req *types.EmbedChannel,
) (*types.EmbedChannel, string, error) {
return nil, "", nil
}
func (f *fakeEmbedChannelService) ListByAgent(
ctx context.Context, tenantID uint64, agentID string,
) ([]*types.EmbedChannel, error) {
return nil, nil
}
func (f *fakeEmbedChannelService) ListByTenant(
ctx context.Context, tenantID uint64,
) ([]*types.EmbedChannel, error) {
return nil, nil
}
func (f *fakeEmbedChannelService) Update(
ctx context.Context, tenantID uint64, id string, req *types.EmbedChannel,
enabled *bool, showSuggested *bool, allowWebSearch *bool, allowMemory *bool, allowFileUpload *bool,
defaultLocale *string, webhookURL *string, webhookSecret *string,
) (*types.EmbedChannel, error) {
return nil, nil
}
func (f *fakeEmbedChannelService) GetOwnedChannel(
ctx context.Context, tenantID uint64, id string,
) (*types.EmbedChannel, error) {
ch := f.channels[id]
if ch == nil || ch.TenantID != tenantID {
return nil, service.ErrEmbedChannelNotFound
}
return ch, nil
}
func (f *fakeEmbedChannelService) Delete(ctx context.Context, tenantID uint64, id string) error {
return nil
}
func (f *fakeEmbedChannelService) RotateToken(
ctx context.Context, tenantID uint64, id string,
) (*types.EmbedChannel, string, error) {
return nil, "", nil
}
func (f *fakeEmbedChannelService) LookupForEmbed(
ctx context.Context, channelID, token string,
) (*types.EmbedChannel, error) {
ch := f.channels[channelID]
if ch == nil || ch.PublishToken != token {
return nil, service.ErrEmbedTokenInvalid
}
if !ch.Enabled {
return nil, service.ErrEmbedChannelDisabled
}
return ch, nil
}
func (f *fakeEmbedChannelService) LookupEnabledChannel(
ctx context.Context, channelID string,
) (*types.EmbedChannel, error) {
ch := f.channels[channelID]
if ch == nil {
return nil, service.ErrEmbedTokenInvalid
}
if !ch.Enabled {
return nil, service.ErrEmbedChannelDisabled
}
return ch, nil
}
func (f *fakeEmbedChannelService) IssueSessionToken(
ctx context.Context, channelID string,
) (string, int, error) {
return "ems_testtoken", 1800, nil
}
func (f *fakeEmbedChannelService) ResolveSessionToken(ctx context.Context, token string) (string, error) {
channelID, ok := f.sessions[token]
if !ok {
return "", service.ErrEmbedTokenInvalid
}
return channelID, nil
}
func (f *fakeEmbedChannelService) PublicConfig(
ctx context.Context, ch *types.EmbedChannel,
) types.EmbedChannelPublicConfig {
return types.EmbedChannelPublicConfig{ChannelID: ch.ID}
}
func (f *fakeEmbedChannelService) SuggestedQuestions(
ctx context.Context, ch *types.EmbedChannel, limit int,
) ([]types.SuggestedQuestion, error) {
return nil, nil
}
func (f *fakeEmbedChannelService) EmbedChunk(
ctx context.Context, ch *types.EmbedChannel, chunkID string,
) (*types.Chunk, error) {
return nil, nil
}
func (f *fakeEmbedChannelService) IssuePreviewSession(
ctx context.Context, tenantID uint64, channelID string,
) (string, int, error) {
return "", 0, nil
}
func (f *fakeEmbedChannelService) EmbedDisplayTitle(ctx context.Context, ch *types.EmbedChannel) string {
return ""
}
type fakeTenantService struct {
tenant *types.Tenant
}
func (f *fakeTenantService) GetTenantByID(ctx context.Context, id uint64) (*types.Tenant, error) {
return f.tenant, nil
}
func (f *fakeTenantService) CreateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) {
return nil, nil
}
func (f *fakeTenantService) GetTenantsByIDs(ctx context.Context, ids []uint64) (map[uint64]*types.Tenant, error) {
return nil, nil
}
func (f *fakeTenantService) UpdateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) {
return nil, nil
}
func (f *fakeTenantService) DeleteTenant(ctx context.Context, id uint64) error {
return nil
}
func (f *fakeTenantService) ListTenants(ctx context.Context) ([]*types.Tenant, error) {
return nil, nil
}
func (f *fakeTenantService) UpdateAPIKey(ctx context.Context, id uint64) (string, error) {
return "", nil
}
func (f *fakeTenantService) ExtractTenantIDFromAPIKey(apiKey string) (uint64, error) {
return 0, nil
}
func (f *fakeTenantService) ListAllTenants(ctx context.Context) ([]*types.Tenant, error) {
return nil, nil
}
func (f *fakeTenantService) BulkSetStorageQuota(ctx context.Context, quotaBytes int64) (int64, error) {
return 0, nil
}
func (f *fakeTenantService) SearchTenants(
ctx context.Context, keyword string, tenantID uint64, page, pageSize int,
) ([]*types.Tenant, int64, error) {
return nil, 0, nil
}
func (f *fakeTenantService) GetTenantByIDForUser(
ctx context.Context, tenantID uint64, userID string,
) (*types.Tenant, error) {
return f.tenant, nil
}
func (f *fakeTenantService) GetWeKnoraCloudCredentials(ctx context.Context) *types.WeKnoraCloudCredentials {
return nil
}
var (
_ interfaces.EmbedChannelService = (*fakeEmbedChannelService)(nil)
_ interfaces.TenantService = (*fakeTenantService)(nil)
)
func TestEmbedGlobalPerMinute(t *testing.T) {
tests := []struct {
perIP int
want int
}{
{perIP: 0, want: 120},
{perIP: 1, want: 120},
{perIP: 6, want: 120},
{perIP: 7, want: 140},
{perIP: 10, want: 200},
}
for _, tt := range tests {
name := fmt.Sprintf("perIP=%d", tt.perIP)
t.Run(name, func(t *testing.T) {
if got := embedGlobalPerMinute(tt.perIP); got != tt.want {
t.Fatalf("embedGlobalPerMinute(%d) = %d, want %d", tt.perIP, got, tt.want)
}
})
}
}
func TestOriginAllowed(t *testing.T) {
tests := []struct {
name string
origin string
allowed []string
want bool
}{
{name: "empty allow list", origin: "https://evil.com", allowed: nil, want: false},
{name: "exact match", origin: "https://app.example.com", allowed: []string{"https://app.example.com"}, want: true},
{name: "wildcard star", origin: "https://any.example.com", allowed: []string{"*"}, want: true},
{name: "subdomain suffix", origin: "https://app.example.com", allowed: []string{"*.example.com"}, want: true},
{name: "missing origin", origin: "", allowed: []string{"https://app.example.com"}, want: false},
{name: "not allowed", origin: "https://evil.com", allowed: []string{"https://app.example.com"}, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := originAllowed(tt.origin, tt.allowed); got != tt.want {
t.Fatalf("originAllowed(%q, %v) = %v, want %v", tt.origin, tt.allowed, got, tt.want)
}
})
}
}
func TestExtractEmbedToken(t *testing.T) {
gin.SetMode(gin.TestMode)
tests := []struct {
name string
header string
query string
want string
}{
{name: "authorization header", header: "Embed em_publish", want: "em_publish"},
{name: "query param rejected", query: "ems_session", want: ""},
{name: "header preferred", header: "Embed em_header", query: "em_query", want: "em_header"},
{name: "missing", want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
req := httptest.NewRequest(http.MethodGet, "/?token="+tt.query, nil)
if tt.header != "" {
req.Header.Set("Authorization", tt.header)
}
c.Request = req
if got := extractEmbedToken(c); got != tt.want {
t.Fatalf("extractEmbedToken() = %q, want %q", got, tt.want)
}
})
}
}
func TestEmbedAuthSessionTokenPath(t *testing.T) {
gin.SetMode(gin.TestMode)
const channelID = "ch-1"
svc := &fakeEmbedChannelService{
channels: map[string]*types.EmbedChannel{
channelID: {
ID: channelID,
TenantID: 42,
Enabled: true,
AllowedOrigins: []byte(`["https://app.example.com"]`),
RateLimitPerMinute: 0,
},
},
sessions: map[string]string{
"ems_valid": channelID,
},
}
tenantSvc := &fakeTenantService{tenant: &types.Tenant{ID: 42}}
r := gin.New()
r.GET("/api/v1/embed/:channel_id/config", EmbedAuth(svc, tenantSvc, nil), func(c *gin.Context) {
ch, ok := EmbedChannelFromContext(c.Request.Context())
if !ok || ch.ID != channelID {
c.JSON(http.StatusInternalServerError, gin.H{"error": "missing channel"})
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
})
req := httptest.NewRequest(http.MethodGet, "/api/v1/embed/"+channelID+"/config", nil)
req.Header.Set("Authorization", "Embed ems_valid")
req.Header.Set("Origin", "https://app.example.com")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("unmarshal response: %v", err)
}
if body["success"] != true {
t.Fatalf("expected success response, got %v", body)
}
}
func TestEmbedAuthPublishTokenValid(t *testing.T) {
gin.SetMode(gin.TestMode)
const (
channelID = "ch-pub-1"
publishToken = "em_publish_ok"
)
svc := &fakeEmbedChannelService{
channels: map[string]*types.EmbedChannel{
channelID: {
ID: channelID,
TenantID: 11,
Enabled: true,
PublishToken: publishToken,
AllowedOrigins: []byte(`["https://app.example.com"]`),
RateLimitPerMinute: 0,
},
},
}
tenantSvc := &fakeTenantService{tenant: &types.Tenant{ID: 11}}
r := gin.New()
r.GET("/api/v1/embed/:channel_id/config", EmbedAuth(svc, tenantSvc, nil), func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"success": true})
})
req := httptest.NewRequest(http.MethodGet, "/api/v1/embed/"+channelID+"/config", nil)
req.Header.Set("Authorization", "Embed "+publishToken)
req.Header.Set("Origin", "https://app.example.com")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
}
func TestEmbedAuthPublishTokenInvalid(t *testing.T) {
gin.SetMode(gin.TestMode)
const channelID = "ch-pub-1"
svc := &fakeEmbedChannelService{
channels: map[string]*types.EmbedChannel{
channelID: {
ID: channelID,
TenantID: 11,
Enabled: true,
PublishToken: "em_real_token",
AllowedOrigins: []byte(`["https://app.example.com"]`),
},
},
}
handler := EmbedAuth(svc, &fakeTenantService{tenant: &types.Tenant{ID: 11}}, nil)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/embed/"+channelID+"/config", nil)
c.Request.Header.Set("Authorization", "Embed em_wrong_token")
c.Request.Header.Set("Origin", "https://app.example.com")
c.Params = gin.Params{{Key: "channel_id", Value: channelID}}
handler(c)
if w.Code != http.StatusUnauthorized {
t.Fatalf("status = %d, want %d, body = %s", w.Code, http.StatusUnauthorized, w.Body.String())
}
}
func TestEmbedAuthSessionTokenMismatch(t *testing.T) {
gin.SetMode(gin.TestMode)
const channelID = "ch-1"
svc := &fakeEmbedChannelService{
sessions: map[string]string{
"ems_other": "other-channel",
},
}
handler := EmbedAuth(svc, &fakeTenantService{tenant: &types.Tenant{ID: 1}}, nil)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/embed/"+channelID+"/config", nil)
c.Request.Header.Set("Authorization", "Embed ems_other")
c.Params = gin.Params{{Key: "channel_id", Value: channelID}}
handler(c)
if w.Code != http.StatusUnauthorized {
t.Fatalf("status = %d, want %d, body = %s", w.Code, http.StatusUnauthorized, w.Body.String())
}
}