fix: SQLite/Lite mode compatibility for FAQ import and chunk operations

Multiple fixes for WeKnora Lite mode (SQLite, no Redis):

1. Redis nil pointer panic on FAQ import (knowledge.go)
   - Add in-memory fallback (sync.Map) for FAQ import progress tracking
   - Add nil guards for running task locks and progress queries

2. seq_id not auto-incrementing in SQLite (chunk.go, tag.go)
   - GORM autoIncrement on non-PK columns doesn't work in SQLite
   - Add AssignChunkSeqIDs() to pre-assign seq_ids before batch insert
   - Add BeforeCreate hook for KnowledgeTag
   - Use Unscoped() to include soft-deleted records in MAX(seq_id) query

3. NOW() function not available in SQLite (chunk.go)
   - Replace raw SQL NOW() with time.Now() or datetime('now')
   - Use dialector check for raw SQL that must differ between PG and SQLite

4. FAQ KB should reject file uploads (knowledge.go)
   - Add kb.Type == "faq" check in CreateKnowledgeFromFile

5. Linux/macOS build compatibility (Makefile)
   - Conditionally apply macOS-only linker flag based on uname

6. Add SQLite integration tests (chunk_sqlite_test.go)
   - TestCreateChunks_SQLite_SeqIDAutoAssigned
   - TestCreateChunks_SQLite_SeqIDContinuesFromExisting
   - TestCreateChunks_SQLite_SeqIDUniqueAcrossKBs
   - TestCreateChunks_SQLite_SeqIDAfterSoftDelete
   - TestKnowledgeTag_SQLite_SeqIDAutoAssigned
   - TestUpdateChunk_SQLite_NoNOWError

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
swim2sun
2026-04-14 17:02:45 +00:00
committed by lyingbug
parent 5000e6e665
commit d2c6814f22
6 changed files with 322 additions and 9 deletions
+2 -2
View File
@@ -231,7 +231,7 @@ build-prod:
COMMIT_ID=$${COMMIT_ID:-unknown}; \
CGO_ENABLED=1 \
CGO_CFLAGS="-Wno-deprecated-declarations" \
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
CGO_LDFLAGS="$$(if [ "$$(uname)" = 'Darwin' ]; then echo '-Wl,-no_warn_duplicate_libraries'; fi)" \
BUILD_TIME=$${BUILD_TIME:-unknown}; \
GO_VERSION=$${GO_VERSION:-unknown}; \
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
@@ -254,7 +254,7 @@ build-lite:
LDFLAGS="$$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
CGO_ENABLED=1 \
CGO_CFLAGS="-Wno-deprecated-declarations" \
CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \
CGO_LDFLAGS="$$(if [ "$$(uname)" = 'Darwin' ]; then echo '-Wl,-no_warn_duplicate_libraries'; fi)" \
go build -tags "sqlite_fts5" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME)-lite $(MAIN_PATH)
# Run Lite version with .env.lite defaults
+16 -6
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/types"
@@ -28,6 +29,11 @@ func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chun
for _, chunk := range chunks {
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
}
// Pre-assign SeqIDs for SQLite compatibility (autoIncrement on non-PK columns
// doesn't work in SQLite). This is a no-op if all chunks already have SeqIDs.
if err := types.AssignChunkSeqIDs(r.db.WithContext(ctx), chunks); err != nil {
return fmt.Errorf("failed to assign chunk seq_ids: %w", err)
}
// Use Select("*") to ensure all fields including zero values (IsEnabled=false, Flags=0)
// are inserted, bypassing GORM's default value behavior for zero values
return r.db.WithContext(ctx).Select("*").CreateInBatches(chunks, 100).Error
@@ -379,7 +385,7 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun
tag_id = CASE %s END,
flags = CASE %s END,
status = CASE %s END,
updated_at = NOW()
updated_at = datetime('now')
WHERE id IN (%s)
`,
strings.Join(contentCases, " "),
@@ -783,14 +789,18 @@ func (r *chunkRepository) UpdateChunkFlagsBatch(
inPlaceholders[i] = "?"
}
nowFunc := "NOW()"
if r.db.Dialector.Name() == "sqlite" {
nowFunc = "datetime('now')"
}
sql := fmt.Sprintf(`
UPDATE chunks
UPDATE chunks
SET flags = (flags | (%s)) & ~(%s),
updated_at = NOW()
WHERE tenant_id = ?
updated_at = %s
WHERE tenant_id = ?
AND knowledge_base_id = ?
AND id IN (%s)
`, setExpr, clearExpr, strings.Join(inPlaceholders, ","))
`, setExpr, clearExpr, nowFunc, strings.Join(inPlaceholders, ","))
args = append(args, tenantID, kbID)
for _, id := range allIDs {
@@ -842,7 +852,7 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
// Build update query
updates := map[string]interface{}{
"updated_at": "NOW()",
"updated_at": time.Now(),
}
if isEnabled != nil {
@@ -0,0 +1,217 @@
package repository
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
// setupChunkTestDB creates an in-memory SQLite database with chunk and tag tables.
func setupChunkTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.KnowledgeTag{}))
return db
}
func makeChunk(kbID, knowledgeID string, chunkType string) *types.Chunk {
return &types.Chunk{
ID: uuid.New().String(),
TenantID: 1,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
Content: "test content",
ChunkType: chunkType,
IsEnabled: true,
}
}
func TestCreateChunks_SQLite_SeqIDAutoAssigned(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
ctx := context.Background()
kbID := uuid.New().String()
knowledgeID := uuid.New().String()
// Create a batch of 5 chunks
chunks := []*types.Chunk{
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
}
err := repo.CreateChunks(ctx, chunks)
require.NoError(t, err)
// Verify all chunks got unique sequential seq_ids
var saved []types.Chunk
require.NoError(t, db.Order("seq_id").Find(&saved).Error)
assert.Len(t, saved, 5)
for i, c := range saved {
assert.Equal(t, int64(i+1), c.SeqID, "chunk %d should have seq_id %d", i, i+1)
}
}
func TestCreateChunks_SQLite_SeqIDContinuesFromExisting(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
ctx := context.Background()
kbID := uuid.New().String()
knowledgeID := uuid.New().String()
// Create first batch
batch1 := []*types.Chunk{
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
}
require.NoError(t, repo.CreateChunks(ctx, batch1))
// Create second batch - seq_ids should continue from 3
batch2 := []*types.Chunk{
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
}
require.NoError(t, repo.CreateChunks(ctx, batch2))
var saved []types.Chunk
require.NoError(t, db.Order("seq_id").Find(&saved).Error)
assert.Len(t, saved, 5)
for i, c := range saved {
assert.Equal(t, int64(i+1), c.SeqID, "chunk %d should have seq_id %d", i, i+1)
}
}
func TestCreateChunks_SQLite_SeqIDUniqueAcrossKBs(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
ctx := context.Background()
kb1 := uuid.New().String()
kb2 := uuid.New().String()
k1 := uuid.New().String()
k2 := uuid.New().String()
// Create chunks in two different knowledge bases
require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{
makeChunk(kb1, k1, "faq"),
makeChunk(kb1, k1, "faq"),
}))
require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{
makeChunk(kb2, k2, "faq"),
makeChunk(kb2, k2, "faq"),
}))
// All seq_ids should be globally unique (1,2,3,4)
var saved []types.Chunk
require.NoError(t, db.Order("seq_id").Find(&saved).Error)
assert.Len(t, saved, 4)
seqIDs := map[int64]bool{}
for _, c := range saved {
assert.NotZero(t, c.SeqID)
assert.False(t, seqIDs[c.SeqID], "seq_id %d should be unique", c.SeqID)
seqIDs[c.SeqID] = true
}
}
func TestKnowledgeTag_SQLite_SeqIDAutoAssigned(t *testing.T) {
db := setupChunkTestDB(t)
ctx := context.Background()
kbID := uuid.New().String()
// Create tags one by one (as the application does)
tag1 := &types.KnowledgeTag{
ID: uuid.New().String(),
TenantID: 1,
KnowledgeBaseID: kbID,
Name: "tag1",
}
tag2 := &types.KnowledgeTag{
ID: uuid.New().String(),
TenantID: 1,
KnowledgeBaseID: kbID,
Name: "tag2",
}
require.NoError(t, db.WithContext(ctx).Create(tag1).Error)
require.NoError(t, db.WithContext(ctx).Create(tag2).Error)
// Both should have non-zero, unique seq_ids
assert.NotZero(t, tag1.SeqID)
assert.NotZero(t, tag2.SeqID)
assert.NotEqual(t, tag1.SeqID, tag2.SeqID)
}
func TestCreateChunks_SQLite_SeqIDAfterSoftDelete(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
ctx := context.Background()
kbID := uuid.New().String()
knowledgeID := uuid.New().String()
// Create first batch
batch1 := []*types.Chunk{
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
}
require.NoError(t, repo.CreateChunks(ctx, batch1))
// Soft-delete all chunks (like frontend "clear" does)
require.NoError(t, db.Where("knowledge_base_id = ?", kbID).Delete(&types.Chunk{}).Error)
// Verify soft-deleted
var activeCount int64
db.Model(&types.Chunk{}).Where("knowledge_base_id = ?", kbID).Count(&activeCount)
assert.Equal(t, int64(0), activeCount, "all chunks should be soft-deleted")
// Create second batch — seq_ids must NOT conflict with soft-deleted ones
batch2 := []*types.Chunk{
makeChunk(kbID, knowledgeID, "faq"),
makeChunk(kbID, knowledgeID, "faq"),
}
err := repo.CreateChunks(ctx, batch2)
require.NoError(t, err, "should not get UNIQUE constraint error after soft delete")
// Verify new seq_ids start after the soft-deleted max (3)
var saved []types.Chunk
require.NoError(t, db.Order("seq_id").Find(&saved).Error)
assert.Len(t, saved, 2)
assert.Equal(t, int64(4), saved[0].SeqID)
assert.Equal(t, int64(5), saved[1].SeqID)
}
func TestUpdateChunk_SQLite_NoNOWError(t *testing.T) {
db := setupChunkTestDB(t)
ctx := context.Background()
kbID := uuid.New().String()
knowledgeID := uuid.New().String()
chunk := makeChunk(kbID, knowledgeID, "faq")
require.NoError(t, db.WithContext(ctx).Create(chunk).Error)
// Test updating a chunk field — verifies no NOW() related errors
err := db.WithContext(ctx).Model(chunk).Update("content", "updated content").Error
assert.NoError(t, err)
var saved types.Chunk
require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error)
assert.Equal(t, "updated content", saved.Content)
}
+34
View File
@@ -78,6 +78,10 @@ type knowledgeService struct {
redisClient *redis.Client
kbShareService interfaces.KBShareService
imageResolver *docparser.ImageResolver
// In-memory fallbacks for Lite mode (no Redis)
memFAQProgress sync.Map // taskID -> *types.FAQImportProgress
memFAQRunningImport sync.Map // kbID -> *runningFAQImportInfo
}
const (
@@ -216,6 +220,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
return nil, err
}
// FAQ knowledge bases should not accept file uploads — use the FAQ import API instead
if kb.Type == types.KnowledgeBaseTypeFAQ {
return nil, werrors.NewBadRequestError("FAQ 知识库不支持文件上传,请使用 FAQ 导入功能")
}
if err := checkStorageEngineConfigured(ctx, kb); err != nil {
return nil, err
}
@@ -6350,6 +6359,12 @@ type runningFAQImportInfo struct {
// getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB
// Returns the task info if found, nil otherwise
func (s *knowledgeService) getRunningFAQImportInfo(ctx context.Context, kbID string) (*runningFAQImportInfo, error) {
if s.redisClient == nil {
if v, ok := s.memFAQRunningImport.Load(kbID); ok {
return v.(*runningFAQImportInfo), nil
}
return nil, nil
}
key := getFAQImportRunningKey(kbID)
data, err := s.redisClient.Get(ctx, key).Result()
if err != nil {
@@ -6383,6 +6398,10 @@ func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID s
// setRunningFAQImportInfo sets the running task info for a KB
func (s *knowledgeService) setRunningFAQImportInfo(ctx context.Context, kbID string, info *runningFAQImportInfo) error {
if s.redisClient == nil {
s.memFAQRunningImport.Store(kbID, info)
return nil
}
key := getFAQImportRunningKey(kbID)
data, err := json.Marshal(info)
if err != nil {
@@ -6393,6 +6412,10 @@ func (s *knowledgeService) setRunningFAQImportInfo(ctx context.Context, kbID str
// clearRunningFAQImportTaskID clears the running task ID for a KB
func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID string) error {
if s.redisClient == nil {
s.memFAQRunningImport.Delete(kbID)
return nil
}
key := getFAQImportRunningKey(kbID)
return s.redisClient.Del(ctx, key).Err()
}
@@ -8490,6 +8513,11 @@ func getFAQImportRunningKey(kbID string) string {
// saveFAQImportProgress saves the FAQ import progress to Redis
func (s *knowledgeService) saveFAQImportProgress(ctx context.Context, progress *types.FAQImportProgress) error {
if s.redisClient == nil {
progress.UpdatedAt = time.Now().Unix()
s.memFAQProgress.Store(progress.TaskID, progress)
return nil
}
key := getFAQImportProgressKey(progress.TaskID)
progress.UpdatedAt = time.Now().Unix()
data, err := json.Marshal(progress)
@@ -8501,6 +8529,12 @@ func (s *knowledgeService) saveFAQImportProgress(ctx context.Context, progress *
// GetFAQImportProgress retrieves the progress of an FAQ import task
func (s *knowledgeService) GetFAQImportProgress(ctx context.Context, taskID string) (*types.FAQImportProgress, error) {
if s.redisClient == nil {
if v, ok := s.memFAQProgress.Load(taskID); ok {
return v.(*types.FAQImportProgress), nil
}
return nil, werrors.NewNotFoundError("FAQ import task not found")
}
key := getFAQImportProgressKey(taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
+31
View File
@@ -162,3 +162,34 @@ type Chunk struct {
// Soft delete marker, supports data recovery
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
}
// AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0.
// Must be called before CreateInBatches for SQLite compatibility.
func AssignChunkSeqIDs(tx *gorm.DB, chunks []*Chunk) error {
needAssign := false
for _, c := range chunks {
if c.SeqID == 0 {
needAssign = true
break
}
}
if !needAssign {
return nil
}
var maxSeqID *int64
if err := tx.Unscoped().Model(&Chunk{}).Select("MAX(seq_id)").Scan(&maxSeqID).Error; err != nil {
return err
}
next := int64(1)
if maxSeqID != nil {
next = *maxSeqID + 1
}
for _, c := range chunks {
if c.SeqID == 0 {
c.SeqID = next
next++
}
}
return nil
}
+22 -1
View File
@@ -1,6 +1,10 @@
package types
import "time"
import (
"time"
"gorm.io/gorm"
)
// KnowledgeTag represents a tag (category) under a specific knowledge base.
// Tags are scoped by knowledge base (and tenant) and are used to categorize
@@ -26,6 +30,23 @@ type KnowledgeTag struct {
UpdatedAt time.Time `json:"updated_at"`
}
// BeforeCreate ensures SeqID is populated for databases that don't support
// autoIncrement on non-primary-key columns (e.g. SQLite).
func (t *KnowledgeTag) BeforeCreate(tx *gorm.DB) error {
if t.SeqID == 0 {
var maxSeqID *int64
tx.Unscoped().Model(&KnowledgeTag{}).
Select("MAX(seq_id)").
Scan(&maxSeqID)
if maxSeqID != nil {
t.SeqID = *maxSeqID + 1
} else {
t.SeqID = 1
}
}
return nil
}
// KnowledgeTagWithStats represents tag information along with usage statistics.
type KnowledgeTagWithStats struct {
KnowledgeTag