feat: 新增异步任务提取服务
This commit is contained in:
@@ -28,6 +28,8 @@ type ChatManage struct {
|
||||
SearchResult []*SearchResult `json:"-"` // Results from search phase
|
||||
RerankResult []*SearchResult `json:"-"` // Results after reranking
|
||||
MergeResult []*SearchResult `json:"-"` // Final merged results after all processing
|
||||
Entity []string `json:"-"` // List of identified entities
|
||||
GraphResult *GraphData `json:"-"` // Graph data from search phase
|
||||
UserContent string `json:"-"` // Processed user content
|
||||
ChatResponse *ChatResponse `json:"-"` // Final response from chat model
|
||||
ResponseChan <-chan StreamResponse `json:"-"` // Channel for streaming responses
|
||||
@@ -75,6 +77,7 @@ const (
|
||||
PREPROCESS_QUERY EventType = "preprocess_query" // Query preprocessing stage
|
||||
REWRITE_QUERY EventType = "rewrite_query" // Query rewriting for better retrieval
|
||||
CHUNK_SEARCH EventType = "chunk_search" // Search for relevant chunks
|
||||
ENTITY_SEARCH EventType = "entity_search" // Search for relevant entities
|
||||
CHUNK_RERANK EventType = "chunk_rerank" // Rerank search results
|
||||
CHUNK_MERGE EventType = "chunk_merge" // Merge similar chunks
|
||||
INTO_CHAT_MESSAGE EventType = "into_chat_message" // Convert chunks into chat messages
|
||||
@@ -104,6 +107,7 @@ var Pipline = map[string][]EventType{
|
||||
REWRITE_QUERY,
|
||||
PREPROCESS_QUERY,
|
||||
CHUNK_SEARCH,
|
||||
ENTITY_SEARCH,
|
||||
CHUNK_RERANK,
|
||||
CHUNK_MERGE,
|
||||
FILTER_TOP_K,
|
||||
|
||||
@@ -19,6 +19,7 @@ const (
|
||||
MatchTypeHistory
|
||||
MatchTypeParentChunk // 父Chunk匹配类型
|
||||
MatchTypeRelationChunk // 关系Chunk匹配类型
|
||||
MatchTypeGraph
|
||||
)
|
||||
|
||||
// IndexInfo contains information about indexed content
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package types
|
||||
|
||||
const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
)
|
||||
|
||||
type ExtractChunkPayload struct {
|
||||
TenantID uint `json:"tenant_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ModelID string `json:"model_id"`
|
||||
}
|
||||
|
||||
type PromptTemplateStructured struct {
|
||||
Description string `json:"description"`
|
||||
Tags []string `json:"tags"`
|
||||
Examples []GraphData `json:"examples"`
|
||||
}
|
||||
|
||||
type GraphNode struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Chunks []string `json:"chunks,omitempty"`
|
||||
Attributes []string `json:"attributes,omitempty"`
|
||||
}
|
||||
|
||||
type GraphRelation struct {
|
||||
Node1 string `json:"node1,omitempty"`
|
||||
Node2 string `json:"node2,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type GraphData struct {
|
||||
Text string `json:"text,omitempty"`
|
||||
Node []*GraphNode `json:"node,omitempty"`
|
||||
Relation []*GraphRelation `json:"relation,omitempty"`
|
||||
}
|
||||
|
||||
type NameSpace struct {
|
||||
KnowledgeBase string `json:"knowledge_base"`
|
||||
Knowledge string `json:"knowledge"`
|
||||
}
|
||||
|
||||
func (n NameSpace) Labels() []string {
|
||||
res := make([]string, 0)
|
||||
if n.KnowledgeBase != "" {
|
||||
res = append(res, n.KnowledgeBase)
|
||||
}
|
||||
if n.Knowledge != "" {
|
||||
res = append(res, n.Knowledge)
|
||||
}
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
type Extracter interface {
|
||||
Extract(ctx context.Context, t *asynq.Task) error
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type RetrieveGraphRepository interface {
|
||||
AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error
|
||||
DelGraph(ctx context.Context, namespace []types.NameSpace) error
|
||||
SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error)
|
||||
}
|
||||
@@ -38,6 +38,8 @@ type KnowledgeBase struct {
|
||||
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
|
||||
// Storage config
|
||||
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
|
||||
// Extract config
|
||||
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
|
||||
// Creation time of the knowledge base
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time of the knowledge base
|
||||
@@ -167,3 +169,27 @@ func (c *VLMConfig) Scan(value interface{}) error {
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
type ExtractConfig struct {
|
||||
Text string `yaml:"text" json:"text"`
|
||||
Tags []string `yaml:"tags" json:"tags"`
|
||||
Nodes []*GraphNode `yaml:"nodes" json:"nodes"`
|
||||
Relations []*GraphRelation `yaml:"relations" json:"relations"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert ExtractConfig to database value
|
||||
func (e ExtractConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface, used to convert database value to ExtractConfig
|
||||
func (e *ExtractConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, e)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user