diff --git a/Makefile b/Makefile index 547906d8..0378d244 100644 --- a/Makefile +++ b/Makefile @@ -231,7 +231,7 @@ build-prod: COMMIT_ID=$${COMMIT_ID:-unknown}; \ CGO_ENABLED=1 \ CGO_CFLAGS="-Wno-deprecated-declarations" \ - CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \ + CGO_LDFLAGS="$$(if [ "$$(uname)" = 'Darwin' ]; then echo '-Wl,-no_warn_duplicate_libraries'; fi)" \ BUILD_TIME=$${BUILD_TIME:-unknown}; \ GO_VERSION=$${GO_VERSION:-unknown}; \ LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \ @@ -254,7 +254,7 @@ build-lite: LDFLAGS="$$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \ CGO_ENABLED=1 \ CGO_CFLAGS="-Wno-deprecated-declarations" \ - CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" \ + CGO_LDFLAGS="$$(if [ "$$(uname)" = 'Darwin' ]; then echo '-Wl,-no_warn_duplicate_libraries'; fi)" \ go build -tags "sqlite_fts5" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME)-lite $(MAIN_PATH) # Run Lite version with .env.lite defaults diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 9abd3837..bc67498e 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/types" @@ -28,6 +29,11 @@ func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chun for _, chunk := range chunks { chunk.Content = common.CleanInvalidUTF8(chunk.Content) } + // Pre-assign SeqIDs for SQLite compatibility (autoIncrement on non-PK columns + // doesn't work in SQLite). This is a no-op if all chunks already have SeqIDs. + if err := types.AssignChunkSeqIDs(r.db.WithContext(ctx), chunks); err != nil { + return fmt.Errorf("failed to assign chunk seq_ids: %w", err) + } // Use Select("*") to ensure all fields including zero values (IsEnabled=false, Flags=0) // are inserted, bypassing GORM's default value behavior for zero values return r.db.WithContext(ctx).Select("*").CreateInBatches(chunks, 100).Error @@ -379,7 +385,7 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun tag_id = CASE %s END, flags = CASE %s END, status = CASE %s END, - updated_at = NOW() + updated_at = datetime('now') WHERE id IN (%s) `, strings.Join(contentCases, " "), @@ -783,14 +789,18 @@ func (r *chunkRepository) UpdateChunkFlagsBatch( inPlaceholders[i] = "?" } + nowFunc := "NOW()" + if r.db.Dialector.Name() == "sqlite" { + nowFunc = "datetime('now')" + } sql := fmt.Sprintf(` - UPDATE chunks + UPDATE chunks SET flags = (flags | (%s)) & ~(%s), - updated_at = NOW() - WHERE tenant_id = ? + updated_at = %s + WHERE tenant_id = ? AND knowledge_base_id = ? AND id IN (%s) -`, setExpr, clearExpr, strings.Join(inPlaceholders, ",")) +`, setExpr, clearExpr, nowFunc, strings.Join(inPlaceholders, ",")) args = append(args, tenantID, kbID) for _, id := range allIDs { @@ -842,7 +852,7 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( // Build update query updates := map[string]interface{}{ - "updated_at": "NOW()", + "updated_at": time.Now(), } if isEnabled != nil { diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.go new file mode 100644 index 00000000..6194d3d5 --- /dev/null +++ b/internal/application/repository/chunk_sqlite_test.go @@ -0,0 +1,217 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// setupChunkTestDB creates an in-memory SQLite database with chunk and tag tables. +func setupChunkTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.Chunk{}, &types.KnowledgeTag{})) + return db +} + +func makeChunk(kbID, knowledgeID string, chunkType string) *types.Chunk { + return &types.Chunk{ + ID: uuid.New().String(), + TenantID: 1, + KnowledgeBaseID: kbID, + KnowledgeID: knowledgeID, + Content: "test content", + ChunkType: chunkType, + IsEnabled: true, + } +} + +func TestCreateChunks_SQLite_SeqIDAutoAssigned(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + kbID := uuid.New().String() + knowledgeID := uuid.New().String() + + // Create a batch of 5 chunks + chunks := []*types.Chunk{ + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + } + + err := repo.CreateChunks(ctx, chunks) + require.NoError(t, err) + + // Verify all chunks got unique sequential seq_ids + var saved []types.Chunk + require.NoError(t, db.Order("seq_id").Find(&saved).Error) + assert.Len(t, saved, 5) + + for i, c := range saved { + assert.Equal(t, int64(i+1), c.SeqID, "chunk %d should have seq_id %d", i, i+1) + } +} + +func TestCreateChunks_SQLite_SeqIDContinuesFromExisting(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + kbID := uuid.New().String() + knowledgeID := uuid.New().String() + + // Create first batch + batch1 := []*types.Chunk{ + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + } + require.NoError(t, repo.CreateChunks(ctx, batch1)) + + // Create second batch - seq_ids should continue from 3 + batch2 := []*types.Chunk{ + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + } + require.NoError(t, repo.CreateChunks(ctx, batch2)) + + var saved []types.Chunk + require.NoError(t, db.Order("seq_id").Find(&saved).Error) + assert.Len(t, saved, 5) + + for i, c := range saved { + assert.Equal(t, int64(i+1), c.SeqID, "chunk %d should have seq_id %d", i, i+1) + } +} + +func TestCreateChunks_SQLite_SeqIDUniqueAcrossKBs(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + kb1 := uuid.New().String() + kb2 := uuid.New().String() + k1 := uuid.New().String() + k2 := uuid.New().String() + + // Create chunks in two different knowledge bases + require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{ + makeChunk(kb1, k1, "faq"), + makeChunk(kb1, k1, "faq"), + })) + require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{ + makeChunk(kb2, k2, "faq"), + makeChunk(kb2, k2, "faq"), + })) + + // All seq_ids should be globally unique (1,2,3,4) + var saved []types.Chunk + require.NoError(t, db.Order("seq_id").Find(&saved).Error) + assert.Len(t, saved, 4) + + seqIDs := map[int64]bool{} + for _, c := range saved { + assert.NotZero(t, c.SeqID) + assert.False(t, seqIDs[c.SeqID], "seq_id %d should be unique", c.SeqID) + seqIDs[c.SeqID] = true + } +} + +func TestKnowledgeTag_SQLite_SeqIDAutoAssigned(t *testing.T) { + db := setupChunkTestDB(t) + ctx := context.Background() + + kbID := uuid.New().String() + + // Create tags one by one (as the application does) + tag1 := &types.KnowledgeTag{ + ID: uuid.New().String(), + TenantID: 1, + KnowledgeBaseID: kbID, + Name: "tag1", + } + tag2 := &types.KnowledgeTag{ + ID: uuid.New().String(), + TenantID: 1, + KnowledgeBaseID: kbID, + Name: "tag2", + } + + require.NoError(t, db.WithContext(ctx).Create(tag1).Error) + require.NoError(t, db.WithContext(ctx).Create(tag2).Error) + + // Both should have non-zero, unique seq_ids + assert.NotZero(t, tag1.SeqID) + assert.NotZero(t, tag2.SeqID) + assert.NotEqual(t, tag1.SeqID, tag2.SeqID) +} + +func TestCreateChunks_SQLite_SeqIDAfterSoftDelete(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + kbID := uuid.New().String() + knowledgeID := uuid.New().String() + + // Create first batch + batch1 := []*types.Chunk{ + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + } + require.NoError(t, repo.CreateChunks(ctx, batch1)) + + // Soft-delete all chunks (like frontend "clear" does) + require.NoError(t, db.Where("knowledge_base_id = ?", kbID).Delete(&types.Chunk{}).Error) + + // Verify soft-deleted + var activeCount int64 + db.Model(&types.Chunk{}).Where("knowledge_base_id = ?", kbID).Count(&activeCount) + assert.Equal(t, int64(0), activeCount, "all chunks should be soft-deleted") + + // Create second batch — seq_ids must NOT conflict with soft-deleted ones + batch2 := []*types.Chunk{ + makeChunk(kbID, knowledgeID, "faq"), + makeChunk(kbID, knowledgeID, "faq"), + } + err := repo.CreateChunks(ctx, batch2) + require.NoError(t, err, "should not get UNIQUE constraint error after soft delete") + + // Verify new seq_ids start after the soft-deleted max (3) + var saved []types.Chunk + require.NoError(t, db.Order("seq_id").Find(&saved).Error) + assert.Len(t, saved, 2) + assert.Equal(t, int64(4), saved[0].SeqID) + assert.Equal(t, int64(5), saved[1].SeqID) +} + +func TestUpdateChunk_SQLite_NoNOWError(t *testing.T) { + db := setupChunkTestDB(t) + ctx := context.Background() + + kbID := uuid.New().String() + knowledgeID := uuid.New().String() + + chunk := makeChunk(kbID, knowledgeID, "faq") + require.NoError(t, db.WithContext(ctx).Create(chunk).Error) + + // Test updating a chunk field — verifies no NOW() related errors + err := db.WithContext(ctx).Model(chunk).Update("content", "updated content").Error + assert.NoError(t, err) + + var saved types.Chunk + require.NoError(t, db.First(&saved, "id = ?", chunk.ID).Error) + assert.Equal(t, "updated content", saved.Content) +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 5d9451de..21d75ae6 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -78,6 +78,10 @@ type knowledgeService struct { redisClient *redis.Client kbShareService interfaces.KBShareService imageResolver *docparser.ImageResolver + + // In-memory fallbacks for Lite mode (no Redis) + memFAQProgress sync.Map // taskID -> *types.FAQImportProgress + memFAQRunningImport sync.Map // kbID -> *runningFAQImportInfo } const ( @@ -216,6 +220,11 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, return nil, err } + // FAQ knowledge bases should not accept file uploads — use the FAQ import API instead + if kb.Type == types.KnowledgeBaseTypeFAQ { + return nil, werrors.NewBadRequestError("FAQ 知识库不支持文件上传,请使用 FAQ 导入功能") + } + if err := checkStorageEngineConfigured(ctx, kb); err != nil { return nil, err } @@ -6350,6 +6359,12 @@ type runningFAQImportInfo struct { // getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB // Returns the task info if found, nil otherwise func (s *knowledgeService) getRunningFAQImportInfo(ctx context.Context, kbID string) (*runningFAQImportInfo, error) { + if s.redisClient == nil { + if v, ok := s.memFAQRunningImport.Load(kbID); ok { + return v.(*runningFAQImportInfo), nil + } + return nil, nil + } key := getFAQImportRunningKey(kbID) data, err := s.redisClient.Get(ctx, key).Result() if err != nil { @@ -6383,6 +6398,10 @@ func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID s // setRunningFAQImportInfo sets the running task info for a KB func (s *knowledgeService) setRunningFAQImportInfo(ctx context.Context, kbID string, info *runningFAQImportInfo) error { + if s.redisClient == nil { + s.memFAQRunningImport.Store(kbID, info) + return nil + } key := getFAQImportRunningKey(kbID) data, err := json.Marshal(info) if err != nil { @@ -6393,6 +6412,10 @@ func (s *knowledgeService) setRunningFAQImportInfo(ctx context.Context, kbID str // clearRunningFAQImportTaskID clears the running task ID for a KB func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID string) error { + if s.redisClient == nil { + s.memFAQRunningImport.Delete(kbID) + return nil + } key := getFAQImportRunningKey(kbID) return s.redisClient.Del(ctx, key).Err() } @@ -8490,6 +8513,11 @@ func getFAQImportRunningKey(kbID string) string { // saveFAQImportProgress saves the FAQ import progress to Redis func (s *knowledgeService) saveFAQImportProgress(ctx context.Context, progress *types.FAQImportProgress) error { + if s.redisClient == nil { + progress.UpdatedAt = time.Now().Unix() + s.memFAQProgress.Store(progress.TaskID, progress) + return nil + } key := getFAQImportProgressKey(progress.TaskID) progress.UpdatedAt = time.Now().Unix() data, err := json.Marshal(progress) @@ -8501,6 +8529,12 @@ func (s *knowledgeService) saveFAQImportProgress(ctx context.Context, progress * // GetFAQImportProgress retrieves the progress of an FAQ import task func (s *knowledgeService) GetFAQImportProgress(ctx context.Context, taskID string) (*types.FAQImportProgress, error) { + if s.redisClient == nil { + if v, ok := s.memFAQProgress.Load(taskID); ok { + return v.(*types.FAQImportProgress), nil + } + return nil, werrors.NewNotFoundError("FAQ import task not found") + } key := getFAQImportProgressKey(taskID) data, err := s.redisClient.Get(ctx, key).Bytes() if err != nil { diff --git a/internal/types/chunk.go b/internal/types/chunk.go index 61326f25..18916d1b 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -162,3 +162,34 @@ type Chunk struct { // Soft delete marker, supports data recovery DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` } + +// AssignChunkSeqIDs assigns sequential SeqIDs to a batch of chunks that have SeqID == 0. +// Must be called before CreateInBatches for SQLite compatibility. +func AssignChunkSeqIDs(tx *gorm.DB, chunks []*Chunk) error { + needAssign := false + for _, c := range chunks { + if c.SeqID == 0 { + needAssign = true + break + } + } + if !needAssign { + return nil + } + + var maxSeqID *int64 + if err := tx.Unscoped().Model(&Chunk{}).Select("MAX(seq_id)").Scan(&maxSeqID).Error; err != nil { + return err + } + next := int64(1) + if maxSeqID != nil { + next = *maxSeqID + 1 + } + for _, c := range chunks { + if c.SeqID == 0 { + c.SeqID = next + next++ + } + } + return nil +} diff --git a/internal/types/tag.go b/internal/types/tag.go index 7bda2e07..75fc2da6 100644 --- a/internal/types/tag.go +++ b/internal/types/tag.go @@ -1,6 +1,10 @@ package types -import "time" +import ( + "time" + + "gorm.io/gorm" +) // KnowledgeTag represents a tag (category) under a specific knowledge base. // Tags are scoped by knowledge base (and tenant) and are used to categorize @@ -26,6 +30,23 @@ type KnowledgeTag struct { UpdatedAt time.Time `json:"updated_at"` } +// BeforeCreate ensures SeqID is populated for databases that don't support +// autoIncrement on non-primary-key columns (e.g. SQLite). +func (t *KnowledgeTag) BeforeCreate(tx *gorm.DB) error { + if t.SeqID == 0 { + var maxSeqID *int64 + tx.Unscoped().Model(&KnowledgeTag{}). + Select("MAX(seq_id)"). + Scan(&maxSeqID) + if maxSeqID != nil { + t.SeqID = *maxSeqID + 1 + } else { + t.SeqID = 1 + } + } + return nil +} + // KnowledgeTagWithStats represents tag information along with usage statistics. type KnowledgeTagWithStats struct { KnowledgeTag