feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
+122
-29
@@ -9,11 +9,13 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo"
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
@@ -25,9 +27,9 @@ import (
|
||||
"go.uber.org/dig"
|
||||
"google.golang.org/grpc"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/client"
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
@@ -36,6 +38,7 @@ import (
|
||||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
sqliteRetrieverRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/sqlite"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
@@ -48,6 +51,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/handler"
|
||||
"github.com/Tencent/WeKnora/internal/handler/session"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/mcp"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
@@ -94,6 +98,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// External service clients
|
||||
logger.Debugf(ctx, "[Container] Registering external service clients...")
|
||||
must(container.Provide(initDocReaderClient))
|
||||
must(container.Provide(initImageResolver))
|
||||
must(container.Provide(initOllamaService))
|
||||
must(container.Provide(initNeo4jClient))
|
||||
must(container.Provide(stream.NewStreamManager))
|
||||
@@ -146,6 +151,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// Extract services - register individual extracters with names
|
||||
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
|
||||
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
|
||||
must(container.Provide(service.NewImageMultimodalService, dig.Name("imageMultimodal")))
|
||||
|
||||
must(container.Provide(service.NewMessageService))
|
||||
must(container.Provide(service.NewMCPServiceService))
|
||||
@@ -169,9 +175,16 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
logger.Debugf(ctx, "[Container] Registering session service...")
|
||||
must(container.Provide(service.NewSessionService))
|
||||
|
||||
logger.Debugf(ctx, "[Container] Registering asynq client and server...")
|
||||
must(container.Provide(router.NewAsyncqClient))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
logger.Debugf(ctx, "[Container] Registering task enqueuer...")
|
||||
redisAvailable := os.Getenv("REDIS_ADDR") != ""
|
||||
if redisAvailable {
|
||||
must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer))))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
} else {
|
||||
syncExec := router.NewSyncTaskExecutor()
|
||||
must(container.Provide(func() interfaces.TaskEnqueuer { return syncExec }))
|
||||
must(container.Provide(func() *router.SyncTaskExecutor { return syncExec }))
|
||||
}
|
||||
|
||||
// Chat pipeline components for processing chat requests
|
||||
logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...")
|
||||
@@ -218,9 +231,13 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
logger.Debugf(ctx, "[Container] HTTP handlers registered")
|
||||
|
||||
// Router configuration
|
||||
logger.Debugf(ctx, "[Container] Registering router and starting asynq server...")
|
||||
logger.Debugf(ctx, "[Container] Registering router and starting task server...")
|
||||
must(container.Provide(router.NewRouter))
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
if redisAvailable {
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
} else {
|
||||
must(container.Invoke(router.RegisterSyncHandlers))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Container] Container initialization completed successfully")
|
||||
return container
|
||||
@@ -249,19 +266,23 @@ func initTracer() (*tracing.Tracer, error) {
|
||||
}
|
||||
|
||||
func initRedisClient() (*redis.Client, error) {
|
||||
redisAddr := os.Getenv("REDIS_ADDR")
|
||||
if redisAddr == "" {
|
||||
logger.Infof(context.Background(), "[Redis] No REDIS_ADDR configured, Redis disabled (Lite mode)")
|
||||
return nil, nil
|
||||
}
|
||||
db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
db = 0
|
||||
}
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: os.Getenv("REDIS_ADDR"),
|
||||
Addr: redisAddr,
|
||||
Username: os.Getenv("REDIS_USERNAME"),
|
||||
Password: os.Getenv("REDIS_PASSWORD"),
|
||||
DB: db,
|
||||
})
|
||||
|
||||
// 验证连接
|
||||
_, err = client.Ping(context.Background()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("连接Redis失败: %w", err)
|
||||
@@ -271,6 +292,10 @@ func initRedisClient() (*redis.Client, error) {
|
||||
}
|
||||
|
||||
func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, error) {
|
||||
if redisClient == nil {
|
||||
logger.Infof(context.Background(), "[ContextStorage] Redis not available, using in-memory storage")
|
||||
return llmcontext.NewMemoryStorage(), nil
|
||||
}
|
||||
storage, err := llmcontext.NewRedisStorage(redisClient, 24*time.Hour, "context:")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -334,6 +359,21 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
os.Getenv("DB_PORT"),
|
||||
os.Getenv("DB_NAME"),
|
||||
)
|
||||
case "sqlite":
|
||||
dbPath := os.Getenv("DB_PATH")
|
||||
if dbPath == "" {
|
||||
dbPath = "./data/weknora.db"
|
||||
}
|
||||
if dir := filepath.Dir(dbPath); dir != "." && dir != "" {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
sqlite_vec.Auto()
|
||||
dsn := dbPath + "?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on"
|
||||
dialector = sqlite.Open(dsn)
|
||||
migrateDSN = "sqlite3://" + dbPath
|
||||
logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER"))
|
||||
}
|
||||
@@ -342,6 +382,16 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if os.Getenv("DB_DRIVER") == "sqlite" {
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get underlying sql.DB: %w", err)
|
||||
}
|
||||
if err := sqlDB.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Run database migrations automatically (optional, can be disabled via env var)
|
||||
// To disable auto-migration, set AUTO_MIGRATE=false
|
||||
// To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true
|
||||
@@ -374,7 +424,14 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
}
|
||||
|
||||
// Configure connection pool parameters
|
||||
sqlDB.SetMaxIdleConns(10)
|
||||
if os.Getenv("DB_DRIVER") == "sqlite" {
|
||||
// SQLite only supports one concurrent writer even in WAL mode.
|
||||
// Limiting to a single open connection serialises all DB access and
|
||||
// prevents "database is locked" errors from concurrent goroutines.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
} else {
|
||||
sqlDB.SetMaxIdleConns(10)
|
||||
}
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute)
|
||||
|
||||
return db, nil
|
||||
@@ -390,7 +447,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
// - Configured file service implementation
|
||||
// - Error if initialization fails
|
||||
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
switch os.Getenv("STORAGE_TYPE") {
|
||||
storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE"))
|
||||
if storageType == "" {
|
||||
storageType = "local"
|
||||
}
|
||||
switch storageType {
|
||||
case "minio":
|
||||
if os.Getenv("MINIO_ENDPOINT") == "" ||
|
||||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" ||
|
||||
@@ -419,8 +480,8 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
os.Getenv("COS_SECRET_ID"),
|
||||
os.Getenv("COS_SECRET_KEY"),
|
||||
os.Getenv("COS_PATH_PREFIX"),
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
||||
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"),
|
||||
os.Getenv("COS_TEMP_REGION"),
|
||||
)
|
||||
case "tos":
|
||||
if os.Getenv("TOS_ENDPOINT") == "" ||
|
||||
@@ -441,11 +502,15 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
os.Getenv("TOS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
)
|
||||
case "local":
|
||||
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
return file.NewLocalFileService(baseDir), nil
|
||||
case "dummy":
|
||||
return file.NewDummyFileService(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE"))
|
||||
return nil, fmt.Errorf("unsupported storage type: %s", storageType)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -474,6 +539,16 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
log.Infof("Register postgres retrieve engine success")
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "sqlite") {
|
||||
sqliteRepo := sqliteRetrieverRepo.NewSQLiteRetrieveEngineRepository(db)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(sqliteRepo, types.SQLiteRetrieverEngineType),
|
||||
); err != nil {
|
||||
log.Errorf("Register sqlite retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register sqlite retrieve engine success")
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "elasticsearch_v8") {
|
||||
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
|
||||
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
|
||||
@@ -639,21 +714,39 @@ func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) {
|
||||
})
|
||||
}
|
||||
|
||||
// initDocReaderClient initializes the document reader client
|
||||
// Creates a client for interacting with the document reader service
|
||||
// Parameters:
|
||||
// - cfg: Application configuration
|
||||
//
|
||||
// Returns:
|
||||
// - Configured document reader client
|
||||
// - Error if initialization fails
|
||||
func initDocReaderClient(cfg *config.Config) (*client.Client, error) {
|
||||
// Use the DocReader URL from environment or config
|
||||
docReaderURL := os.Getenv("DOCREADER_ADDR")
|
||||
if docReaderURL == "" && cfg.DocReader != nil {
|
||||
docReaderURL = cfg.DocReader.Addr
|
||||
// initDocReaderClient initializes the DocumentReader client (lightweight API).
|
||||
func initDocReaderClient(cfg *config.Config) (interfaces.DocumentReader, error) {
|
||||
addr := strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
|
||||
transport := strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
|
||||
if transport == "" {
|
||||
transport = "grpc"
|
||||
}
|
||||
return client.NewClient(docReaderURL)
|
||||
if addr == "" {
|
||||
logger.Infof(context.Background(), "[DocConverter] No DOCREADER_ADDR configured, starting disconnected")
|
||||
}
|
||||
transport = strings.ToLower(transport)
|
||||
switch transport {
|
||||
case "http", "https":
|
||||
if addr != "" && !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
|
||||
addr = "http://" + addr
|
||||
}
|
||||
return docparser.NewHTTPDocumentReader(addr)
|
||||
default:
|
||||
return docparser.NewGRPCDocumentReader(addr)
|
||||
}
|
||||
}
|
||||
|
||||
// initImageResolver creates the ImageResolver for storing images from DocReader.
|
||||
func initImageResolver(cfg *config.Config) *docparser.ImageResolver {
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
baseDir = "/data/files"
|
||||
}
|
||||
urlPrefix := os.Getenv("LOCAL_STORAGE_URL_PREFIX")
|
||||
if urlPrefix == "" {
|
||||
urlPrefix = "/files"
|
||||
}
|
||||
return docparser.NewImageResolver(baseDir, urlPrefix)
|
||||
}
|
||||
|
||||
// initOllamaService initializes the Ollama service client
|
||||
|
||||
Reference in New Issue
Block a user