feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+122 -29
View File
@@ -9,11 +9,13 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
_ "github.com/duckdb/duckdb-go/v2"
esv7 "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v8"
@@ -25,9 +27,9 @@ import (
"go.uber.org/dig"
"google.golang.org/grpc"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"github.com/Tencent/WeKnora/docreader/client"
"github.com/Tencent/WeKnora/internal/application/repository"
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
@@ -36,6 +38,7 @@ import (
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
sqliteRetrieverRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/sqlite"
"github.com/Tencent/WeKnora/internal/application/service"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/application/service/file"
@@ -48,6 +51,7 @@ import (
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/handler"
"github.com/Tencent/WeKnora/internal/handler/session"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/mcp"
"github.com/Tencent/WeKnora/internal/models/embedding"
@@ -94,6 +98,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// External service clients
logger.Debugf(ctx, "[Container] Registering external service clients...")
must(container.Provide(initDocReaderClient))
must(container.Provide(initImageResolver))
must(container.Provide(initOllamaService))
must(container.Provide(initNeo4jClient))
must(container.Provide(stream.NewStreamManager))
@@ -146,6 +151,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
// Extract services - register individual extracters with names
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
must(container.Provide(service.NewMessageService))
must(container.Provide(service.NewMCPServiceService))
@@ -169,9 +175,16 @@ func BuildContainer(container *dig.Container) *dig.Container {
logger.Debugf(ctx, "[Container] Registering session service...")
must(container.Provide(service.NewSessionService))
logger.Debugf(ctx, "[Container] Registering asynq client and server...")
must(container.Provide(router.NewAsyncqClient))
must(container.Provide(router.NewAsynqServer))
logger.Debugf(ctx, "[Container] Registering task enqueuer...")
redisAvailable := os.Getenv("REDIS_ADDR") != ""
if redisAvailable {
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
must(container.Provide(router.NewAsynqServer))
} else {
syncExec := router.NewSyncTaskExecutor()
must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec }))
must(container.Provide(func() *router.SyncTaskExecutor { return syncExec }))
}
// Chat pipeline components for processing chat requests
logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...")
@@ -218,9 +231,13 @@ func BuildContainer(container *dig.Container) *dig.Container {
logger.Debugf(ctx, "[Container] HTTP handlers registered")
// Router configuration
logger.Debugf(ctx, "[Container] Registering router and starting asynq server...")
logger.Debugf(ctx, "[Container] Registering router and starting task server...")
must(container.Provide(router.NewRouter))
must(container.Invoke(router.RunAsynqServer))
if redisAvailable {
must(container.Invoke(router.RunAsynqServer))
} else {
must(container.Invoke(router.RegisterSyncHandlers))
}
logger.Infof(ctx, "[Container] Container initialization completed successfully")
return container
@@ -249,19 +266,23 @@ func initTracer() (*tracing.Tracer, error) {
}
func initRedisClient() (*redis.Client, error) {
redisAddr := os.Getenv("REDIS_ADDR")
if redisAddr == "" {
logger.Infof(context.Background(), "[Redis] No REDIS_ADDR configured, Redis disabled (Lite mode)")
return nil, nil
}
db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
if err != nil {
return nil, err
db = 0
}
client := redis.NewClient(&redis.Options{
Addr: os.Getenv("REDIS_ADDR"),
Addr: redisAddr,
Username: os.Getenv("REDIS_USERNAME"),
Password: os.Getenv("REDIS_PASSWORD"),
DB: db,
})
// 验证连接
_, err = client.Ping(context.Background()).Result()
if err != nil {
return nil, fmt.Errorf("连接Redis失败: %w", err)
@@ -271,6 +292,10 @@ func initRedisClient() (*redis.Client, error) {
}
func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, error) {
if redisClient == nil {
logger.Infof(context.Background(), "[ContextStorage] Redis not available, using in-memory storage")
return llmcontext.NewMemoryStorage(), nil
}
storage, err := llmcontext.NewRedisStorage(redisClient, 24*time.Hour, "context:")
if err != nil {
return nil, err
@@ -334,6 +359,21 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
os.Getenv("DB_PORT"),
os.Getenv("DB_NAME"),
)
case "sqlite":
dbPath := os.Getenv("DB_PATH")
if dbPath == "" {
dbPath = "./data/weknora.db"
}
if dir := filepath.Dir(dbPath); dir != "." && dir != "" {
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err)
}
}
sqlite_vec.Auto()
dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on"
dialector = sqlite.Open(dsn)
migrateDSN = "sqlite3://" + dbPath
logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath)
default:
return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER"))
}
@@ -342,6 +382,16 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
return nil, err
}
if os.Getenv("DB_DRIVER") == "sqlite" {
sqlDB, err := db.DB()
if err != nil {
return nil, fmt.Errorf("failed to get underlying sql.DB: %w", err)
}
if err := sqlDB.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
}
}
// Run database migrations automatically (optional, can be disabled via env var)
// To disable auto-migration, set AUTO_MIGRATE=false
// To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true
@@ -374,7 +424,14 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
}
// Configure connection pool parameters
sqlDB.SetMaxIdleConns(10)
if os.Getenv("DB_DRIVER") == "sqlite" {
// SQLite only supports one concurrent writer even in WAL mode.
// Limiting to a single open connection serialises all DB access and
// prevents "database is locked" errors from concurrent goroutines.
sqlDB.SetMaxOpenConns(1)
} else {
sqlDB.SetMaxIdleConns(10)
}
sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute)
return db, nil
@@ -390,7 +447,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
// - Configured file service implementation
// - Error if initialization fails
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
switch os.Getenv("STORAGE_TYPE") {
storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE"))
if storageType == "" {
storageType = "local"
}
switch storageType {
case "minio":
if os.Getenv("MINIO_ENDPOINT") == "" ||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" ||
@@ -419,8 +480,8 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("COS_SECRET_ID"),
os.Getenv("COS_SECRET_KEY"),
os.Getenv("COS_PATH_PREFIX"),
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
os.Getenv("COS_TEMP_BUCKET_NAME"),
os.Getenv("COS_TEMP_REGION"),
)
case "tos":
if os.Getenv("TOS_ENDPOINT") == "" ||
@@ -441,11 +502,15 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
)
case "local":
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
}
return file.NewLocalFileService(baseDir), nil
case "dummy":
return file.NewDummyFileService(), nil
default:
return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE"))
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
}
}
@@ -474,6 +539,16 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
log.Infof("Register postgres retrieve engine success")
}
}
if slices.Contains(retrieveDriver, "sqlite") {
sqliteRepo := sqliteRetrieverRepo.NewSQLiteRetrieveEngineRepository(db)
if err := registry.Register(
retriever.NewKVHybridRetrieveEngine(sqliteRepo, types.SQLiteRetrieverEngineType),
); err != nil {
log.Errorf("Register sqlite retrieve engine failed: %v", err)
} else {
log.Infof("Register sqlite retrieve engine success")
}
}
if slices.Contains(retrieveDriver, "elasticsearch_v8") {
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
@@ -639,21 +714,39 @@ func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) {
})
}
// initDocReaderClient initializes the document reader client
// Creates a client for interacting with the document reader service
// Parameters:
// - cfg: Application configuration
//
// Returns:
// - Configured document reader client
// - Error if initialization fails
func initDocReaderClient(cfg *config.Config) (*client.Client, error) {
// Use the DocReader URL from environment or config
docReaderURL := os.Getenv("DOCREADER_ADDR")
if docReaderURL == "" && cfg.DocReader != nil {
docReaderURL = cfg.DocReader.Addr
// initDocReaderClient initializes the DocumentReader client (lightweight API).
func initDocReaderClient(cfg *config.Config) (interfaces.DocumentReader, error) {
addr := strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
transport := strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
if transport == "" {
transport = "grpc"
}
return client.NewClient(docReaderURL)
if addr == "" {
logger.Infof(context.Background(), "[DocConverter] No DOCREADER_ADDR configured, starting disconnected")
}
transport = strings.ToLower(transport)
switch transport {
case "http", "https":
if addr != "" && !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
addr = "http://" + addr
}
return docparser.NewHTTPDocumentReader(addr)
default:
return docparser.NewGRPCDocumentReader(addr)
}
}
// initImageResolver creates the ImageResolver for storing images from DocReader.
func initImageResolver(cfg *config.Config) *docparser.ImageResolver {
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
baseDir = "/data/files"
}
urlPrefix := os.Getenv("LOCAL_STORAGE_URL_PREFIX")
if urlPrefix == "" {
urlPrefix = "/files"
}
return docparser.NewImageResolver(baseDir, urlPrefix)
}
// initOllamaService initializes the Ollama service client