diff --git a/.env.example b/.env.example index a17e78f4..4faea879 100644 --- a/.env.example +++ b/.env.example @@ -210,6 +210,15 @@ DB_PASSWORD=postgres123!@# # 数据库名称 DB_NAME=WeKnora +# PostgreSQL 全局最大连接数(Compose 内置 PostgreSQL 服务) +POSTGRES_MAX_CONNECTIONS=300 + +# WeKnora database/sql 连接池 +DB_MAX_OPEN_CONNS=120 +DB_MAX_IDLE_CONNS=20 +DB_CONN_MAX_IDLE_TIME=5m +DB_CONN_MAX_LIFETIME=10m + # 如果使用 redis 作为流处理后端,需要配置以下参数 # Redis用户名,Redis 6.0+ ACL 功能支持(可选) # REDIS_USERNAME= diff --git a/docker-compose.yml b/docker-compose.yml index ce77c40f..02122311 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,10 @@ services: - DB_USER=${DB_USER:-} - DB_PASSWORD=${DB_PASSWORD:-} - DB_NAME=${DB_NAME:-} + - DB_MAX_OPEN_CONNS=${DB_MAX_OPEN_CONNS:-120} + - DB_MAX_IDLE_CONNS=${DB_MAX_IDLE_CONNS:-20} + - DB_CONN_MAX_IDLE_TIME=${DB_CONN_MAX_IDLE_TIME:-5m} + - DB_CONN_MAX_LIFETIME=${DB_CONN_MAX_LIFETIME:-10m} - TZ=${TZ:-Asia/Shanghai} - WEKNORA_LANGUAGE=${WEKNORA_LANGUAGE:-zh-CN} # ========== Langfuse (optional observability) ========== @@ -291,6 +295,7 @@ services: postgres: image: paradedb/paradedb:v0.22.2-pg17 container_name: WeKnora-postgres + command: ["postgres", "-c", "max_connections=${POSTGRES_MAX_CONNECTIONS:-300}"] environment: - POSTGRES_USER=${DB_USER} - POSTGRES_PASSWORD=${DB_PASSWORD} diff --git a/internal/container/container.go b/internal/container/container.go index 758039ff..ff1531e0 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -90,6 +90,62 @@ import ( wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc" ) +func nonNegativeEnvInt(name string, fallback int) int { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + + value, err := strconv.Atoi(raw) + if err != nil || value < 0 { + return fallback + } + return value +} + +func nonNegativeEnvDuration(name string, fallback time.Duration) time.Duration { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + if raw == "0" { + return 0 + } + + value, err := time.ParseDuration(raw) + if err != nil || value < 0 { + return fallback + } + return value +} + +type databasePoolConfig struct { + maxOpen int + maxIdle int + maxIdleTime time.Duration + maxLifetime time.Duration +} + +func databasePoolConfigFromEnv() databasePoolConfig { + cfg := databasePoolConfig{ + maxOpen: nonNegativeEnvInt("DB_MAX_OPEN_CONNS", 0), + maxIdle: nonNegativeEnvInt("DB_MAX_IDLE_CONNS", 10), + maxIdleTime: nonNegativeEnvDuration("DB_CONN_MAX_IDLE_TIME", 0), + maxLifetime: nonNegativeEnvDuration("DB_CONN_MAX_LIFETIME", 10*time.Minute), + } + if cfg.maxOpen > 0 && cfg.maxIdle > cfg.maxOpen { + cfg.maxIdle = cfg.maxOpen + } + return cfg +} + +func applyDatabasePoolConfig(db *sql.DB, cfg databasePoolConfig) { + db.SetMaxOpenConns(cfg.maxOpen) + db.SetMaxIdleConns(cfg.maxIdle) + db.SetConnMaxIdleTime(cfg.maxIdleTime) + db.SetConnMaxLifetime(cfg.maxLifetime) +} + // BuildContainer constructs the dependency injection container // Registers all components, services, repositories and handlers needed by the application // Creates a fully configured application container with proper dependency resolution @@ -608,10 +664,19 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { // Limiting to a single open connection serialises all DB access and // prevents "database is locked" errors from concurrent goroutines. sqlDB.SetMaxOpenConns(1) + sqlDB.SetConnMaxLifetime(10 * time.Minute) } else { - sqlDB.SetMaxIdleConns(10) + poolCfg := databasePoolConfigFromEnv() + applyDatabasePoolConfig(sqlDB, poolCfg) + logger.Infof( + context.Background(), + "Database pool configured: max_open=%d max_idle=%d max_idle_time=%s max_lifetime=%s", + poolCfg.maxOpen, + poolCfg.maxIdle, + poolCfg.maxIdleTime, + poolCfg.maxLifetime, + ) } - sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute) return db, nil } diff --git a/internal/container/db_pool_config_test.go b/internal/container/db_pool_config_test.go new file mode 100644 index 00000000..77cb7aaa --- /dev/null +++ b/internal/container/db_pool_config_test.go @@ -0,0 +1,105 @@ +package container + +import ( + "database/sql" + "testing" + "time" +) + +func TestNonNegativeEnvInt(t *testing.T) { + tests := []struct { + name string + raw string + fallback int + want int + }{ + {name: "uses fallback when unset", raw: "", fallback: 10, want: 10}, + {name: "accepts positive value", raw: "120", fallback: 10, want: 120}, + {name: "accepts zero", raw: "0", fallback: 10, want: 0}, + {name: "rejects negative value", raw: "-1", fallback: 10, want: 10}, + {name: "rejects invalid value", raw: "many", fallback: 10, want: 10}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("TEST_DB_POOL_INT", tt.raw) + if got := nonNegativeEnvInt("TEST_DB_POOL_INT", tt.fallback); got != tt.want { + t.Fatalf("nonNegativeEnvInt() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestNonNegativeEnvDuration(t *testing.T) { + tests := []struct { + name string + raw string + fallback time.Duration + want time.Duration + }{ + {name: "uses fallback when unset", raw: "", fallback: 10 * time.Minute, want: 10 * time.Minute}, + {name: "accepts duration", raw: "5m", fallback: 10 * time.Minute, want: 5 * time.Minute}, + {name: "accepts zero", raw: "0", fallback: 10 * time.Minute, want: 0}, + {name: "rejects negative duration", raw: "-1m", fallback: 10 * time.Minute, want: 10 * time.Minute}, + {name: "rejects invalid duration", raw: "later", fallback: 10 * time.Minute, want: 10 * time.Minute}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("TEST_DB_POOL_DURATION", tt.raw) + if got := nonNegativeEnvDuration("TEST_DB_POOL_DURATION", tt.fallback); got != tt.want { + t.Fatalf("nonNegativeEnvDuration() = %s, want %s", got, tt.want) + } + }) + } +} + +func TestDatabasePoolConfigFromEnv(t *testing.T) { + t.Setenv("DB_MAX_OPEN_CONNS", "120") + t.Setenv("DB_MAX_IDLE_CONNS", "20") + t.Setenv("DB_CONN_MAX_IDLE_TIME", "5m") + t.Setenv("DB_CONN_MAX_LIFETIME", "10m") + + got := databasePoolConfigFromEnv() + if got.maxOpen != 120 { + t.Fatalf("maxOpen = %d, want 120", got.maxOpen) + } + if got.maxIdle != 20 { + t.Fatalf("maxIdle = %d, want 20", got.maxIdle) + } + if got.maxIdleTime != 5*time.Minute { + t.Fatalf("maxIdleTime = %s, want 5m", got.maxIdleTime) + } + if got.maxLifetime != 10*time.Minute { + t.Fatalf("maxLifetime = %s, want 10m", got.maxLifetime) + } +} + +func TestDatabasePoolConfigClampsIdleToOpen(t *testing.T) { + t.Setenv("DB_MAX_OPEN_CONNS", "5") + t.Setenv("DB_MAX_IDLE_CONNS", "20") + + got := databasePoolConfigFromEnv() + if got.maxIdle != 5 { + t.Fatalf("maxIdle = %d, want 5", got.maxIdle) + } +} + +func TestApplyDatabasePoolConfig(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("sql.Open() error = %v", err) + } + defer db.Close() + + applyDatabasePoolConfig(db, databasePoolConfig{ + maxOpen: 120, + maxIdle: 20, + maxIdleTime: 5 * time.Minute, + maxLifetime: 10 * time.Minute, + }) + + if got := db.Stats().MaxOpenConnections; got != 120 { + t.Fatalf("MaxOpenConnections = %d, want 120", got) + } +}