feat: 新增异步任务提取服务

This commit is contained in:
begoniezhao
2025-09-24 12:15:17 +08:00
parent 31d19095f4
commit b6d8be634a
30 changed files with 2514 additions and 117 deletions
+4
View File
@@ -28,6 +28,8 @@ type ChatManage struct {
SearchResult []*SearchResult `json:"-"` // Results from search phase
RerankResult []*SearchResult `json:"-"` // Results after reranking
MergeResult []*SearchResult `json:"-"` // Final merged results after all processing
Entity []string `json:"-"` // List of identified entities
GraphResult *GraphData `json:"-"` // Graph data from search phase
UserContent string `json:"-"` // Processed user content
ChatResponse *ChatResponse `json:"-"` // Final response from chat model
ResponseChan <-chan StreamResponse `json:"-"` // Channel for streaming responses
@@ -75,6 +77,7 @@ const (
PREPROCESS_QUERY EventType = "preprocess_query" // Query preprocessing stage
REWRITE_QUERY EventType = "rewrite_query" // Query rewriting for better retrieval
CHUNK_SEARCH EventType = "chunk_search" // Search for relevant chunks
ENTITY_SEARCH EventType = "entity_search" // Search for relevant entities
CHUNK_RERANK EventType = "chunk_rerank" // Rerank search results
CHUNK_MERGE EventType = "chunk_merge" // Merge similar chunks
INTO_CHAT_MESSAGE EventType = "into_chat_message" // Convert chunks into chat messages
@@ -104,6 +107,7 @@ var Pipline = map[string][]EventType{
REWRITE_QUERY,
PREPROCESS_QUERY,
CHUNK_SEARCH,
ENTITY_SEARCH,
CHUNK_RERANK,
CHUNK_MERGE,
FILTER_TOP_K,
+1
View File
@@ -19,6 +19,7 @@ const (
MatchTypeHistory
MatchTypeParentChunk // 父Chunk匹配类型
MatchTypeRelationChunk // 关系Chunk匹配类型
MatchTypeGraph
)
// IndexInfo contains information about indexed content
+51
View File
@@ -0,0 +1,51 @@
package types
const (
TypeChunkExtract = "chunk:extract"
)
type ExtractChunkPayload struct {
TenantID uint `json:"tenant_id"`
ChunkID string `json:"chunk_id"`
ModelID string `json:"model_id"`
}
type PromptTemplateStructured struct {
Description string `json:"description"`
Tags []string `json:"tags"`
Examples []GraphData `json:"examples"`
}
type GraphNode struct {
Name string `json:"name,omitempty"`
Chunks []string `json:"chunks,omitempty"`
Attributes []string `json:"attributes,omitempty"`
}
type GraphRelation struct {
Node1 string `json:"node1,omitempty"`
Node2 string `json:"node2,omitempty"`
Type string `json:"type,omitempty"`
}
type GraphData struct {
Text string `json:"text,omitempty"`
Node []*GraphNode `json:"node,omitempty"`
Relation []*GraphRelation `json:"relation,omitempty"`
}
type NameSpace struct {
KnowledgeBase string `json:"knowledge_base"`
Knowledge string `json:"knowledge"`
}
func (n NameSpace) Labels() []string {
res := make([]string, 0)
if n.KnowledgeBase != "" {
res = append(res, n.KnowledgeBase)
}
if n.Knowledge != "" {
res = append(res, n.Knowledge)
}
return res
}
+11
View File
@@ -0,0 +1,11 @@
package interfaces
import (
"context"
"github.com/hibiken/asynq"
)
type Extracter interface {
Extract(ctx context.Context, t *asynq.Task) error
}
@@ -0,0 +1,13 @@
package interfaces
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
)
type RetrieveGraphRepository interface {
AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error
DelGraph(ctx context.Context, namespace []types.NameSpace) error
SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error)
}
+26
View File
@@ -38,6 +38,8 @@ type KnowledgeBase struct {
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
// Storage config
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
// Extract config
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
// Creation time of the knowledge base
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
// Last updated time of the knowledge base
@@ -167,3 +169,27 @@ func (c *VLMConfig) Scan(value interface{}) error {
}
return json.Unmarshal(b, c)
}
type ExtractConfig struct {
Text string `yaml:"text" json:"text"`
Tags []string `yaml:"tags" json:"tags"`
Nodes []*GraphNode `yaml:"nodes" json:"nodes"`
Relations []*GraphRelation `yaml:"relations" json:"relations"`
}
// Value implements the driver.Valuer interface, used to convert ExtractConfig to database value
func (e ExtractConfig) Value() (driver.Value, error) {
return json.Marshal(e)
}
// Scan implements the sql.Scanner interface, used to convert database value to ExtractConfig
func (e *ExtractConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(b, e)
}